%load_ext autoreload
%autoreload 2
The autoreload extension is already loaded. To reload it, use: %reload_ext autoreload
from pathlib import Path
import sys
source_dir = str(Path.cwd().parent)
if source_dir not in sys.path:
sys.path.append(source_dir)
import pandas as pd
import numpy as np
import matplotlib.pyplot as plt
import plotly.express as px
import plotly.graph_objects as go
from plotly.subplots import make_subplots
from sklearn.model_selection import TimeSeriesSplit # https://scikit-learn.org/stable/modules/generated/sklearn.model_selection.TimeSeriesSplit.html
from prophet import Prophet
from prophet.plot import plot_plotly, plot_components_plotly, plot_cross_validation_metric
from prophet.diagnostics import cross_validation, performance_metrics
import itertools
pd.options.display.float_format = '{:,.2f}'.format
from oranges_case_lab.configs import config
from oranges_case_lab.constants import constants
import oranges_case_lab.utils
# loading both prices and GoogleData csv files
prices_orig = pd.read_csv(str(constants.DATASETS_DIR) + "/prices.csv")
google_orig = pd.read_csv(str(constants.DATASETS_DIR) + "/GoogleData_oranges.csv")
# overview of the prices dataset structure, its head
prices_orig.head()
| Date | AveragePrice | TotalVolume | 4046 | 4225 | 4770 | TotalBags | SmallBags | LargeBags | XLargeBags | type | year | region | |
|---|---|---|---|---|---|---|---|---|---|---|---|---|---|
| 0 | 2015-01-04 | 1.22 | 40,873.28 | 2,819.50 | 28,287.42 | 49.90 | 9,716.46 | 9,186.93 | 529.53 | 0.00 | conventional | 2015 | Albany |
| 1 | 2015-01-04 | 1.00 | 435,021.49 | 364,302.39 | 23,821.16 | 82.15 | 46,815.79 | 16,707.15 | 30,108.64 | 0.00 | conventional | 2015 | Atlanta |
| 2 | 2015-01-04 | NaN | 788,025.06 | 53,987.31 | 552,906.04 | 39,995.03 | 141,136.68 | 137,146.07 | 3,990.61 | 0.00 | conventional | 2015 | BaltimoreWashington |
| 3 | 2015-01-04 | 1.01 | 80,034.32 | 44,562.12 | 24,964.23 | 2,752.35 | 7,755.62 | 6,064.30 | 1,691.32 | 0.00 | conventional | 2015 | Boise |
| 4 | 2015-01-04 | 1.02 | 491,738.00 | 7,193.87 | 396,752.18 | 128.82 | 87,663.13 | 87,406.84 | 256.29 | 0.00 | conventional | 2015 | Boston |
# basic info of the prices dataset
prices_orig.info()
<class 'pandas.core.frame.DataFrame'> RangeIndex: 25161 entries, 0 to 25160 Data columns (total 13 columns): # Column Non-Null Count Dtype --- ------ -------------- ----- 0 Date 25161 non-null object 1 AveragePrice 24259 non-null float64 2 TotalVolume 24212 non-null float64 3 4046 24261 non-null float64 4 4225 24255 non-null float64 5 4770 24245 non-null float64 6 TotalBags 24271 non-null float64 7 SmallBags 24227 non-null float64 8 LargeBags 24214 non-null float64 9 XLargeBags 24293 non-null float64 10 type 25161 non-null object 11 year 25161 non-null int64 12 region 25161 non-null object dtypes: float64(9), int64(1), object(3) memory usage: 2.5+ MB
# changing the datatype of the date column
prices_orig = prices_orig.astype({'Date': 'datetime64[ns]'})
# prices dataset overview
prices_orig.describe(include="all")
/Users/radimmusalek/opt/anaconda3/lib/python3.7/site-packages/ipykernel_launcher.py:2: FutureWarning: Treating datetime data as categorical rather than numeric in `.describe` is deprecated and will be removed in a future version of pandas. Specify `datetime_is_numeric=True` to silence this warning and adopt the future behavior now.
| Date | AveragePrice | TotalVolume | 4046 | 4225 | 4770 | TotalBags | SmallBags | LargeBags | XLargeBags | type | year | region | |
|---|---|---|---|---|---|---|---|---|---|---|---|---|---|
| count | 25161 | 24,259.00 | 24,212.00 | 24,261.00 | 24,255.00 | 24,245.00 | 24,271.00 | 24,227.00 | 24,214.00 | 24,293.00 | 25161 | 25,161.00 | 25161 |
| unique | 233 | NaN | NaN | NaN | NaN | NaN | NaN | NaN | NaN | NaN | 2 | NaN | 54 |
| top | 2016-10-23 00:00:00 | NaN | NaN | NaN | NaN | NaN | NaN | NaN | NaN | NaN | conventional | NaN | Columbus |
| freq | 108 | NaN | NaN | NaN | NaN | NaN | NaN | NaN | NaN | NaN | 12582 | NaN | 466 |
| first | 2015-01-04 00:00:00 | NaN | NaN | NaN | NaN | NaN | NaN | NaN | NaN | NaN | NaN | NaN | NaN |
| last | 2019-07-14 00:00:00 | NaN | NaN | NaN | NaN | NaN | NaN | NaN | NaN | NaN | NaN | NaN | NaN |
| mean | NaN | 1.40 | 917,091.34 | 298,153.02 | 295,828.43 | 22,463.62 | 295,043.21 | 212,419.88 | 75,899.24 | 4,405.64 | NaN | 2,016.78 | NaN |
| std | NaN | 0.38 | 3,731,937.46 | 1,288,583.40 | 1,209,444.82 | 104,662.07 | 1,232,967.42 | 869,304.26 | 358,607.93 | 25,806.50 | NaN | 1.32 | NaN |
| min | NaN | 0.44 | 84.56 | 0.00 | 0.00 | 0.00 | 0.00 | 0.00 | 0.00 | 0.00 | NaN | 2,015.00 | NaN |
| 25% | NaN | 1.11 | 13,234.04 | 822.59 | 3,061.14 | 0.00 | 7,319.51 | 4,770.44 | 235.82 | 0.00 | NaN | 2,016.00 | NaN |
| 50% | NaN | 1.36 | 119,052.77 | 9,855.11 | 26,977.55 | 193.76 | 46,257.71 | 31,337.34 | 4,043.68 | 0.00 | NaN | 2,017.00 | NaN |
| 75% | NaN | 1.64 | 473,378.30 | 114,121.88 | 149,871.91 | 6,078.17 | 139,429.54 | 100,882.73 | 30,503.37 | 313.89 | NaN | 2,018.00 | NaN |
| max | NaN | 3.25 | 63,716,144.15 | 22,743,616.17 | 20,470,572.61 | 2,546,439.11 | 23,472,988.69 | 15,436,246.72 | 7,864,297.23 | 844,929.83 | NaN | 2,019.00 | NaN |
# interactive plot with conventional type prices per region over the time period
fig_pr_con = px.line(prices_orig[prices_orig["type"]=="conventional"],
x="Date", y="AveragePrice",
color="region", title='Average price of conventional per region')
fig_pr_con.update_xaxes(
dtick="M6",
tickformat="%b-%y")
fig_pr_con.show()
# interactive plot with organic type prices per region over the time period
fig_pr_org = px.line(prices_orig[prices_orig["type"]=="organic"],
x="Date", y="AveragePrice",
color="region", title='Average price of organic per region')
fig_pr_org.update_xaxes(
dtick="M6",
tickformat="%b-%y")
fig_pr_org.show()
# interactive plot with conventional type volumes per region over the time period
fig_vol_con = px.line(prices_orig[prices_orig["type"]=="conventional"],
x="Date", y="TotalVolume",
color="region", title="Total volumes of conventional per region")
fig_vol_con.update_xaxes(
dtick="M6",
tickformat="%b-%y")
fig_vol_con.show()
# interactive plot with organic type volumes per region over the time period
fig_vol_org = px.line(prices_orig[prices_orig["type"]=="organic"],
x="Date", y="TotalVolume",
color="region", title="Total volumes of organic per region")
fig_vol_org.update_xaxes(
dtick="M6",
tickformat="%b-%y")
fig_vol_org.show()
# overview of the Google dataset structure, its head
google_orig.head()
| Unnamed: 0 | oranges: (United States) | organic: (United States) | oranges recipe: (United States) | oranges smoothie: (United States) | oranges salad: (United States) | organic oranges: (United States) | |
|---|---|---|---|---|---|---|---|
| 0 | 2015-01-04 | 46 | 76 | 84 | 8 | 51 | 10 |
| 1 | 2015-01-11 | 48 | 79 | 76 | 8 | 38 | 29 |
| 2 | 2015-01-18 | 48 | 82 | 84 | 9 | 66 | 27 |
| 3 | 2015-01-25 | 49 | 82 | 89 | 8 | 44 | 28 |
| 4 | 2015-02-01 | 52 | 81 | 82 | 7 | 46 | 42 |
# basic info about the Google dataset
google_orig.info()
<class 'pandas.core.frame.DataFrame'> RangeIndex: 239 entries, 0 to 238 Data columns (total 7 columns): # Column Non-Null Count Dtype --- ------ -------------- ----- 0 Unnamed: 0 239 non-null object 1 oranges: (United States) 239 non-null int64 2 organic: (United States) 239 non-null int64 3 oranges recipe: (United States) 239 non-null int64 4 oranges smoothie: (United States) 239 non-null int64 5 oranges salad: (United States) 239 non-null int64 6 organic oranges: (United States) 239 non-null int64 dtypes: int64(6), object(1) memory usage: 13.2+ KB
# changing the datatype of the date column
google_orig = google_orig.astype({"Unnamed: 0": "datetime64[ns]"})
# Google dataset overview
google_orig.describe(include="all")
/Users/radimmusalek/opt/anaconda3/lib/python3.7/site-packages/ipykernel_launcher.py:2: FutureWarning: Treating datetime data as categorical rather than numeric in `.describe` is deprecated and will be removed in a future version of pandas. Specify `datetime_is_numeric=True` to silence this warning and adopt the future behavior now.
| Unnamed: 0 | oranges: (United States) | organic: (United States) | oranges recipe: (United States) | oranges smoothie: (United States) | oranges salad: (United States) | organic oranges: (United States) | |
|---|---|---|---|---|---|---|---|
| count | 239 | 239.00 | 239.00 | 239.00 | 239.00 | 239.00 | 239.00 |
| unique | 239 | NaN | NaN | NaN | NaN | NaN | NaN |
| top | 2018-08-19 00:00:00 | NaN | NaN | NaN | NaN | NaN | NaN |
| freq | 1 | NaN | NaN | NaN | NaN | NaN | NaN |
| first | 2015-01-04 00:00:00 | NaN | NaN | NaN | NaN | NaN | NaN |
| last | 2019-07-28 00:00:00 | NaN | NaN | NaN | NaN | NaN | NaN |
| mean | NaN | 52.31 | 82.50 | 66.99 | 32.71 | 50.18 | 36.06 |
| std | NaN | 11.29 | 6.85 | 14.84 | 19.54 | 12.26 | 12.70 |
| min | NaN | 29.00 | 56.00 | 33.00 | 7.00 | 15.00 | 10.00 |
| 25% | NaN | 44.50 | 79.00 | 55.00 | 15.00 | 42.00 | 27.00 |
| 50% | NaN | 52.00 | 83.00 | 68.00 | 30.00 | 50.00 | 35.00 |
| 75% | NaN | 60.00 | 87.00 | 78.50 | 50.50 | 59.00 | 44.50 |
| max | NaN | 100.00 | 100.00 | 100.00 | 100.00 | 100.00 | 100.00 |
# renaming the columns to get more readable format
google_orig.columns = ["Date", "oranges", "organic",
"oranges recipe", "oranges smoothie",
"oranges salad", "organic oranges"]
# interactive plot with G-searches per keyword over the time period
fig3 = px.line(google_orig, x="Date", y=google_orig.columns,
title="Google searches overview", labels=dict(value="Relative number of searches"))
fig3.update_xaxes(
dtick="M6",
tickformat="%b-%y")
fig3.show()
# split the prices dataset into organic and conventional type
prices_US_conv = prices_orig[prices_orig["type"]=="conventional"].reset_index(drop=True)
prices_US_org = prices_orig[prices_orig["type"]=="organic"].reset_index(drop=True)
# grouping the regional data of the conventional type oranges into whole US by Date
# AveragePrice is mean of its values, the other columns are summarised
prices_US_conv_group = prices_US_conv.groupby("Date").agg({"AveragePrice":"mean",
"TotalVolume":"sum",
"4046":"sum",
"4225":"sum",
"4770":"sum",
"TotalBags":"sum",
"SmallBags":"sum",
"LargeBags":"sum",
"XLargeBags":"sum"
}).reset_index()
# same process as above done for the organic oranges data
prices_US_org_group = prices_US_org.groupby("Date").agg({"AveragePrice":"mean",
"TotalVolume":"sum",
"4046":"sum",
"4225":"sum",
"4770":"sum",
"TotalBags":"sum",
"SmallBags":"sum",
"LargeBags":"sum",
"XLargeBags":"sum"
}).reset_index()
# merging of the both grouped data with the google searches data
# we keep all the columns of the google dataset for both merges
prices_US_conv_merged = prices_US_conv_group.merge(google_orig, how="left", on="Date")
prices_US_org_merged = prices_US_org_group.merge(google_orig, how="left", on="Date")
# left join is used as the most significant parameters for the prediction will be the price and volumes
# since those are the ones we're interested in for our model
# one missing date for G-searches, no need to extrapolate since Prophet doesn't care,
# otherwise I could e.g. mean of the values around that day
# spitting the datasets on training part (till the end of 2018) and testing part (from beginning of 2019)
prices_US_conv_2015_2018 = prices_US_conv_merged[prices_US_conv_merged["Date"]<"2019-01-01"]
prices_US_conv_2019 = prices_US_conv_merged[prices_US_conv_merged["Date"]>="2019-01-01"]
prices_US_org_2015_2018 = prices_US_org_merged[prices_US_org_merged["Date"]<"2019-01-01"]
prices_US_org_2019 = prices_US_org_merged[prices_US_org_merged["Date"]>="2019-01-01"]
# double-check that the split is done correctly
print(prices_US_conv_2015_2018.iloc[-1][0])
print(prices_US_conv_2019.iloc[0][0])
print(prices_US_org_2015_2018.iloc[-1][0])
print(prices_US_org_2019.iloc[0][0])
2018-12-02 00:00:00 2019-01-07 00:00:00 2018-12-02 00:00:00 2019-01-07 00:00:00
# simplifing the dataframe for use by Prophet
conv_P_df = prices_US_conv_2015_2018[["Date", "AveragePrice"]]
conv_P_df.columns = ["ds", "y"]
# creating model with default values, skipping only the daily and weekly seasonality which isn't present in our data
model_conv_P_def = Prophet(daily_seasonality=False, weekly_seasonality=False)
# fitting the model on the whole training set
model_conv_P_def.fit(conv_P_df)
<prophet.forecaster.Prophet at 0x7fdcf51f9090>
future_conv_P_def = model_conv_P_def.make_future_dataframe(periods=len(prices_US_conv_2019), freq="W")
# forecast visualisation
forecast_conv_P_def = model_conv_P_def.predict(future_conv_P_def)
fig_conv_P_def = model_conv_P_def.plot(forecast_conv_P_def)
# model components visualisation
fig_conv_P_comp_def = model_conv_P_def.plot_components(forecast_conv_P_def)
# selecting only the test period forecast
pred_conv_P_2019_def = forecast_conv_P_def[forecast_conv_P_def["ds"]>"2019-01-01"][["ds", "yhat"]]
# simplifing the actuals dataframe for comparison
actuals_conv_P_2019_def = prices_US_conv_2019[["Date", "AveragePrice"]]
actuals_conv_P_2019_def.columns = ["ds", "y"]
# merging the prediction and actuals dataframes into one
conv_P_2019_def = actuals_conv_P_2019_def.merge(pred_conv_P_2019_def, on="ds")
# plotting training, test and prediction data
conv_P_df.set_index("ds")["y"].plot(label="training")
conv_P_2019_def.set_index("ds")["y"].plot(label="test")
conv_P_2019_def.set_index("ds")["yhat"].plot(label="prediction")
plt.xlabel("Date")
plt.ylabel("Price")
plt.title("Price conventional - default model")
plt.legend()
plt.savefig(str(constants.OUTPUTS_DIR) + "/Price conventional - default model.png")
plt.show()
# Defining MAPE function
def MAPE(Y_actual,Y_Predicted):
mape = np.mean(np.abs((Y_actual - Y_Predicted)/Y_actual))*100
return mape
# model's MAPE calculation
model_conv_P_def_MAPE = MAPE(conv_P_2019_def["y"], conv_P_2019_def["yhat"])
print("Default model MAPE: {:.2f}".format(model_conv_P_def_MAPE))
Default model MAPE: 18.27
# cross-validation hyperparameters on the Prophet model
param_grid = {
"changepoint_prior_scale": [0.001, 0.01, 0.1, 0.5],
"seasonality_prior_scale": [0.01, 0.1, 1.0, 10.0],
"seasonality_mode": ["additive", "multiplicative"],
"growth": ["linear", "flat"],
"daily_seasonality": [False],
"weekly_seasonality": [False]
}
# Generate all combinations of parameters
all_params = [dict(zip(param_grid.keys(), v)) for v in itertools.product(*param_grid.values())]
mapes = [] # Store the MAPEs for each params here
# Use cross validation to evaluate all parameters
for params in all_params:
m = Prophet(**params).fit(conv_P_df) # Fit model with given params
df_cv_conv_P = cross_validation(m, initial="730 days", period="90 days", horizon="365 days", parallel="processes")
df_p_conv_P = performance_metrics(df_cv_conv_P, rolling_window=1)
mapes.append(df_p_conv_P["mape"].values[0])
# Find the best parameters
tuning_results = pd.DataFrame(all_params)
tuning_results["mape"] = mapes
print(tuning_results)
INFO:prophet:Making 4 forecasts with cutoffs between 2017-03-07 00:00:00 and 2017-12-02 00:00:00 INFO:prophet:Applying in parallel with <concurrent.futures.process.ProcessPoolExecutor object at 0x7fdcf5ccf350> WARNING:prophet.models:Optimization terminated abnormally. Falling back to Newton. WARNING:prophet.models:Optimization terminated abnormally. Falling back to Newton. INFO:prophet:Making 4 forecasts with cutoffs between 2017-03-07 00:00:00 and 2017-12-02 00:00:00 INFO:prophet:Applying in parallel with <concurrent.futures.process.ProcessPoolExecutor object at 0x7fdcf7abba10> INFO:prophet:Making 4 forecasts with cutoffs between 2017-03-07 00:00:00 and 2017-12-02 00:00:00 INFO:prophet:Applying in parallel with <concurrent.futures.process.ProcessPoolExecutor object at 0x7fdcf7b67450> WARNING:prophet.models:Optimization terminated abnormally. Falling back to Newton. WARNING:prophet.models:Optimization terminated abnormally. Falling back to Newton. INFO:prophet:Making 4 forecasts with cutoffs between 2017-03-07 00:00:00 and 2017-12-02 00:00:00 INFO:prophet:Applying in parallel with <concurrent.futures.process.ProcessPoolExecutor object at 0x7fdcf7b72210> INFO:prophet:Making 4 forecasts with cutoffs between 2017-03-07 00:00:00 and 2017-12-02 00:00:00 INFO:prophet:Applying in parallel with <concurrent.futures.process.ProcessPoolExecutor object at 0x7fdcf7ad5b50> WARNING:prophet.models:Optimization terminated abnormally. Falling back to Newton. WARNING:prophet.models:Optimization terminated abnormally. Falling back to Newton. INFO:prophet:Making 4 forecasts with cutoffs between 2017-03-07 00:00:00 and 2017-12-02 00:00:00 INFO:prophet:Applying in parallel with <concurrent.futures.process.ProcessPoolExecutor object at 0x7fdcf5ccfcd0> INFO:prophet:Making 4 forecasts with cutoffs between 2017-03-07 00:00:00 and 2017-12-02 00:00:00 INFO:prophet:Applying in parallel with <concurrent.futures.process.ProcessPoolExecutor object at 0x7fdcf7b72510> WARNING:prophet.models:Optimization terminated abnormally. Falling back to Newton. WARNING:prophet.models:Optimization terminated abnormally. Falling back to Newton. INFO:prophet:Making 4 forecasts with cutoffs between 2017-03-07 00:00:00 and 2017-12-02 00:00:00 INFO:prophet:Applying in parallel with <concurrent.futures.process.ProcessPoolExecutor object at 0x7fdcf5ccf150> INFO:prophet:Making 4 forecasts with cutoffs between 2017-03-07 00:00:00 and 2017-12-02 00:00:00 INFO:prophet:Applying in parallel with <concurrent.futures.process.ProcessPoolExecutor object at 0x7fdcf7b72410> WARNING:prophet.models:Optimization terminated abnormally. Falling back to Newton. WARNING:prophet.models:Optimization terminated abnormally. Falling back to Newton. INFO:prophet:Making 4 forecasts with cutoffs between 2017-03-07 00:00:00 and 2017-12-02 00:00:00 INFO:prophet:Applying in parallel with <concurrent.futures.process.ProcessPoolExecutor object at 0x7fdcf7ad5a90> INFO:prophet:Making 4 forecasts with cutoffs between 2017-03-07 00:00:00 and 2017-12-02 00:00:00 INFO:prophet:Applying in parallel with <concurrent.futures.process.ProcessPoolExecutor object at 0x7fdcf7b72350> WARNING:prophet.models:Optimization terminated abnormally. Falling back to Newton. WARNING:prophet.models:Optimization terminated abnormally. Falling back to Newton. INFO:prophet:Making 4 forecasts with cutoffs between 2017-03-07 00:00:00 and 2017-12-02 00:00:00 INFO:prophet:Applying in parallel with <concurrent.futures.process.ProcessPoolExecutor object at 0x7fdcf5ccf2d0> INFO:prophet:Making 4 forecasts with cutoffs between 2017-03-07 00:00:00 and 2017-12-02 00:00:00 INFO:prophet:Applying in parallel with <concurrent.futures.process.ProcessPoolExecutor object at 0x7fdcf7c43550> WARNING:prophet.models:Optimization terminated abnormally. Falling back to Newton. WARNING:prophet.models:Optimization terminated abnormally. Falling back to Newton. INFO:prophet:Making 4 forecasts with cutoffs between 2017-03-07 00:00:00 and 2017-12-02 00:00:00 INFO:prophet:Applying in parallel with <concurrent.futures.process.ProcessPoolExecutor object at 0x7fdcf7b72b50> INFO:prophet:Making 4 forecasts with cutoffs between 2017-03-07 00:00:00 and 2017-12-02 00:00:00 INFO:prophet:Applying in parallel with <concurrent.futures.process.ProcessPoolExecutor object at 0x7fdcf7b67390> WARNING:prophet.models:Optimization terminated abnormally. Falling back to Newton. WARNING:prophet.models:Optimization terminated abnormally. Falling back to Newton. INFO:prophet:Making 4 forecasts with cutoffs between 2017-03-07 00:00:00 and 2017-12-02 00:00:00 INFO:prophet:Applying in parallel with <concurrent.futures.process.ProcessPoolExecutor object at 0x7fdcf7b67c90> INFO:prophet:Making 4 forecasts with cutoffs between 2017-03-07 00:00:00 and 2017-12-02 00:00:00 INFO:prophet:Applying in parallel with <concurrent.futures.process.ProcessPoolExecutor object at 0x7fdcf7a99f10> INFO:prophet:Making 4 forecasts with cutoffs between 2017-03-07 00:00:00 and 2017-12-02 00:00:00 INFO:prophet:Applying in parallel with <concurrent.futures.process.ProcessPoolExecutor object at 0x7fdcf7b89ad0> INFO:prophet:Making 4 forecasts with cutoffs between 2017-03-07 00:00:00 and 2017-12-02 00:00:00 INFO:prophet:Applying in parallel with <concurrent.futures.process.ProcessPoolExecutor object at 0x7fdcf7b72490> INFO:prophet:Making 4 forecasts with cutoffs between 2017-03-07 00:00:00 and 2017-12-02 00:00:00 INFO:prophet:Applying in parallel with <concurrent.futures.process.ProcessPoolExecutor object at 0x7fdcf7b72750> INFO:prophet:Making 4 forecasts with cutoffs between 2017-03-07 00:00:00 and 2017-12-02 00:00:00 INFO:prophet:Applying in parallel with <concurrent.futures.process.ProcessPoolExecutor object at 0x7fdcf7b8de90> INFO:prophet:Making 4 forecasts with cutoffs between 2017-03-07 00:00:00 and 2017-12-02 00:00:00 INFO:prophet:Applying in parallel with <concurrent.futures.process.ProcessPoolExecutor object at 0x7fdcf7b72890> INFO:prophet:Making 4 forecasts with cutoffs between 2017-03-07 00:00:00 and 2017-12-02 00:00:00 INFO:prophet:Applying in parallel with <concurrent.futures.process.ProcessPoolExecutor object at 0x7fdcf5ccf2d0> INFO:prophet:Making 4 forecasts with cutoffs between 2017-03-07 00:00:00 and 2017-12-02 00:00:00 INFO:prophet:Applying in parallel with <concurrent.futures.process.ProcessPoolExecutor object at 0x7fdcf7c4bc90> INFO:prophet:Making 4 forecasts with cutoffs between 2017-03-07 00:00:00 and 2017-12-02 00:00:00 INFO:prophet:Applying in parallel with <concurrent.futures.process.ProcessPoolExecutor object at 0x7fdcf7ad55d0> INFO:prophet:Making 4 forecasts with cutoffs between 2017-03-07 00:00:00 and 2017-12-02 00:00:00 INFO:prophet:Applying in parallel with <concurrent.futures.process.ProcessPoolExecutor object at 0x7fdcf59fefd0> INFO:prophet:Making 4 forecasts with cutoffs between 2017-03-07 00:00:00 and 2017-12-02 00:00:00 INFO:prophet:Applying in parallel with <concurrent.futures.process.ProcessPoolExecutor object at 0x7fdcf7b89ad0> INFO:prophet:Making 4 forecasts with cutoffs between 2017-03-07 00:00:00 and 2017-12-02 00:00:00 INFO:prophet:Applying in parallel with <concurrent.futures.process.ProcessPoolExecutor object at 0x7fdcf7c43890> INFO:prophet:Making 4 forecasts with cutoffs between 2017-03-07 00:00:00 and 2017-12-02 00:00:00 INFO:prophet:Applying in parallel with <concurrent.futures.process.ProcessPoolExecutor object at 0x7fdcf7c78450> INFO:prophet:Making 4 forecasts with cutoffs between 2017-03-07 00:00:00 and 2017-12-02 00:00:00 INFO:prophet:Applying in parallel with <concurrent.futures.process.ProcessPoolExecutor object at 0x7fdcf7c60f50> INFO:prophet:Making 4 forecasts with cutoffs between 2017-03-07 00:00:00 and 2017-12-02 00:00:00 INFO:prophet:Applying in parallel with <concurrent.futures.process.ProcessPoolExecutor object at 0x7fdcf7b8d410> INFO:prophet:Making 4 forecasts with cutoffs between 2017-03-07 00:00:00 and 2017-12-02 00:00:00 INFO:prophet:Applying in parallel with <concurrent.futures.process.ProcessPoolExecutor object at 0x7fdcf7c559d0> INFO:prophet:Making 4 forecasts with cutoffs between 2017-03-07 00:00:00 and 2017-12-02 00:00:00 INFO:prophet:Applying in parallel with <concurrent.futures.process.ProcessPoolExecutor object at 0x7fdcf7c45c10> INFO:prophet:Making 4 forecasts with cutoffs between 2017-03-07 00:00:00 and 2017-12-02 00:00:00 INFO:prophet:Applying in parallel with <concurrent.futures.process.ProcessPoolExecutor object at 0x7fdcf7c71490> INFO:prophet:Making 4 forecasts with cutoffs between 2017-03-07 00:00:00 and 2017-12-02 00:00:00 INFO:prophet:Applying in parallel with <concurrent.futures.process.ProcessPoolExecutor object at 0x7fdcf7ad5850> INFO:prophet:Making 4 forecasts with cutoffs between 2017-03-07 00:00:00 and 2017-12-02 00:00:00 INFO:prophet:Applying in parallel with <concurrent.futures.process.ProcessPoolExecutor object at 0x7fdcf7c60f50> INFO:prophet:Making 4 forecasts with cutoffs between 2017-03-07 00:00:00 and 2017-12-02 00:00:00 INFO:prophet:Applying in parallel with <concurrent.futures.process.ProcessPoolExecutor object at 0x7fdcf7c78510> INFO:prophet:Making 4 forecasts with cutoffs between 2017-03-07 00:00:00 and 2017-12-02 00:00:00 INFO:prophet:Applying in parallel with <concurrent.futures.process.ProcessPoolExecutor object at 0x7fdcf7c45590> INFO:prophet:Making 4 forecasts with cutoffs between 2017-03-07 00:00:00 and 2017-12-02 00:00:00 INFO:prophet:Applying in parallel with <concurrent.futures.process.ProcessPoolExecutor object at 0x7fdcf7c79c10> INFO:prophet:Making 4 forecasts with cutoffs between 2017-03-07 00:00:00 and 2017-12-02 00:00:00 INFO:prophet:Applying in parallel with <concurrent.futures.process.ProcessPoolExecutor object at 0x7fdcf7c60e50> INFO:prophet:Making 4 forecasts with cutoffs between 2017-03-07 00:00:00 and 2017-12-02 00:00:00 INFO:prophet:Applying in parallel with <concurrent.futures.process.ProcessPoolExecutor object at 0x7fdcf7c76e10> INFO:prophet:Making 4 forecasts with cutoffs between 2017-03-07 00:00:00 and 2017-12-02 00:00:00 INFO:prophet:Applying in parallel with <concurrent.futures.process.ProcessPoolExecutor object at 0x7fdcf7c4b390> INFO:prophet:Making 4 forecasts with cutoffs between 2017-03-07 00:00:00 and 2017-12-02 00:00:00 INFO:prophet:Applying in parallel with <concurrent.futures.process.ProcessPoolExecutor object at 0x7fdcf59a8550> INFO:prophet:Making 4 forecasts with cutoffs between 2017-03-07 00:00:00 and 2017-12-02 00:00:00 INFO:prophet:Applying in parallel with <concurrent.futures.process.ProcessPoolExecutor object at 0x7fdcf7c718d0> INFO:prophet:Making 4 forecasts with cutoffs between 2017-03-07 00:00:00 and 2017-12-02 00:00:00 INFO:prophet:Applying in parallel with <concurrent.futures.process.ProcessPoolExecutor object at 0x7fdcf7c45c10> INFO:prophet:Making 4 forecasts with cutoffs between 2017-03-07 00:00:00 and 2017-12-02 00:00:00 INFO:prophet:Applying in parallel with <concurrent.futures.process.ProcessPoolExecutor object at 0x7fdcf7c57250> INFO:prophet:Making 4 forecasts with cutoffs between 2017-03-07 00:00:00 and 2017-12-02 00:00:00 INFO:prophet:Applying in parallel with <concurrent.futures.process.ProcessPoolExecutor object at 0x7fdcf7c5fb10> INFO:prophet:Making 4 forecasts with cutoffs between 2017-03-07 00:00:00 and 2017-12-02 00:00:00 INFO:prophet:Applying in parallel with <concurrent.futures.process.ProcessPoolExecutor object at 0x7fdcf7ae2210> INFO:prophet:Making 4 forecasts with cutoffs between 2017-03-07 00:00:00 and 2017-12-02 00:00:00 INFO:prophet:Applying in parallel with <concurrent.futures.process.ProcessPoolExecutor object at 0x7fdcf7c45c10> INFO:prophet:Making 4 forecasts with cutoffs between 2017-03-07 00:00:00 and 2017-12-02 00:00:00 INFO:prophet:Applying in parallel with <concurrent.futures.process.ProcessPoolExecutor object at 0x7fdcf7c79c10> INFO:prophet:Making 4 forecasts with cutoffs between 2017-03-07 00:00:00 and 2017-12-02 00:00:00 INFO:prophet:Applying in parallel with <concurrent.futures.process.ProcessPoolExecutor object at 0x7fdcf7b8d410> INFO:prophet:Making 4 forecasts with cutoffs between 2017-03-07 00:00:00 and 2017-12-02 00:00:00 INFO:prophet:Applying in parallel with <concurrent.futures.process.ProcessPoolExecutor object at 0x7fdcf7c6cb10> INFO:prophet:Making 4 forecasts with cutoffs between 2017-03-07 00:00:00 and 2017-12-02 00:00:00 INFO:prophet:Applying in parallel with <concurrent.futures.process.ProcessPoolExecutor object at 0x7fdcf7c79f10> INFO:prophet:Making 4 forecasts with cutoffs between 2017-03-07 00:00:00 and 2017-12-02 00:00:00 INFO:prophet:Applying in parallel with <concurrent.futures.process.ProcessPoolExecutor object at 0x7fdcf7ade3d0> INFO:prophet:Making 4 forecasts with cutoffs between 2017-03-07 00:00:00 and 2017-12-02 00:00:00 INFO:prophet:Applying in parallel with <concurrent.futures.process.ProcessPoolExecutor object at 0x7fdcf7c601d0> INFO:prophet:Making 4 forecasts with cutoffs between 2017-03-07 00:00:00 and 2017-12-02 00:00:00 INFO:prophet:Applying in parallel with <concurrent.futures.process.ProcessPoolExecutor object at 0x7fdcf7c74310> INFO:prophet:Making 4 forecasts with cutoffs between 2017-03-07 00:00:00 and 2017-12-02 00:00:00 INFO:prophet:Applying in parallel with <concurrent.futures.process.ProcessPoolExecutor object at 0x7fdcf7ade290> INFO:prophet:Making 4 forecasts with cutoffs between 2017-03-07 00:00:00 and 2017-12-02 00:00:00 INFO:prophet:Applying in parallel with <concurrent.futures.process.ProcessPoolExecutor object at 0x7fdcf59fefd0> INFO:prophet:Making 4 forecasts with cutoffs between 2017-03-07 00:00:00 and 2017-12-02 00:00:00 INFO:prophet:Applying in parallel with <concurrent.futures.process.ProcessPoolExecutor object at 0x7fdcf7c6c450> INFO:prophet:Making 4 forecasts with cutoffs between 2017-03-07 00:00:00 and 2017-12-02 00:00:00 INFO:prophet:Applying in parallel with <concurrent.futures.process.ProcessPoolExecutor object at 0x7fdcf7ade510> INFO:prophet:Making 4 forecasts with cutoffs between 2017-03-07 00:00:00 and 2017-12-02 00:00:00 INFO:prophet:Applying in parallel with <concurrent.futures.process.ProcessPoolExecutor object at 0x7fdcf7c6cb10> INFO:prophet:Making 4 forecasts with cutoffs between 2017-03-07 00:00:00 and 2017-12-02 00:00:00 INFO:prophet:Applying in parallel with <concurrent.futures.process.ProcessPoolExecutor object at 0x7fdcf7ae5e50> INFO:prophet:Making 4 forecasts with cutoffs between 2017-03-07 00:00:00 and 2017-12-02 00:00:00 INFO:prophet:Applying in parallel with <concurrent.futures.process.ProcessPoolExecutor object at 0x7fdcf7b67850> INFO:prophet:Making 4 forecasts with cutoffs between 2017-03-07 00:00:00 and 2017-12-02 00:00:00 INFO:prophet:Applying in parallel with <concurrent.futures.process.ProcessPoolExecutor object at 0x7fdcf7ae8350>
changepoint_prior_scale seasonality_prior_scale seasonality_mode growth \
0 0.00 0.01 additive linear
1 0.00 0.01 additive flat
2 0.00 0.01 multiplicative linear
3 0.00 0.01 multiplicative flat
4 0.00 0.10 additive linear
.. ... ... ... ...
59 0.50 1.00 multiplicative flat
60 0.50 10.00 additive linear
61 0.50 10.00 additive flat
62 0.50 10.00 multiplicative linear
63 0.50 10.00 multiplicative flat
daily_seasonality weekly_seasonality mape
0 False False 0.16
1 False False 0.11
2 False False 0.16
3 False False 0.11
4 False False 0.15
.. ... ... ...
59 False False 0.11
60 False False 0.23
61 False False 0.11
62 False False 0.30
63 False False 0.11
[64 rows x 7 columns]
# selecting the best hyperparameters
best_params_conv_P = all_params[np.argmin(mapes)]
print(best_params_conv_P)
{'changepoint_prior_scale': 0.001, 'seasonality_prior_scale': 0.01, 'seasonality_mode': 'additive', 'growth': 'flat', 'daily_seasonality': False, 'weekly_seasonality': False}
# creating the model with the best parameters
model_conv_P = Prophet(**best_params_conv_P)
# fitting the model on the whole training set
model_conv_P.fit(conv_P_df)
<prophet.forecaster.Prophet at 0x7fdcf7aafa10>
future_conv_P = model_conv_P.make_future_dataframe(periods=len(prices_US_conv_2019), freq="W")
# forecast visualisation
forecast_conv_P = model_conv_P.predict(future_conv_P)
fig_conv_P = model_conv_P.plot(forecast_conv_P)
# model components visualisation
fig_conv_P_comp = model_conv_P.plot_components(forecast_conv_P)
# cross-validation of the selected model
df_cv_conv_P = cross_validation(model_conv_P, initial="730 days", period="90 days", horizon="365 days", parallel="processes")
INFO:prophet:Making 4 forecasts with cutoffs between 2017-03-07 00:00:00 and 2017-12-02 00:00:00 INFO:prophet:Applying in parallel with <concurrent.futures.process.ProcessPoolExecutor object at 0x7fdcf829c1d0>
# MAPE of the cross-validated model visualisation
fig_conv_P_cv_mape = plot_cross_validation_metric(df_cv_conv_P, metric='mape')
# selecting only the test period forecast
pred_conv_P_2019 = forecast_conv_P[forecast_conv_P["ds"]>"2019-01-01"][["ds", "yhat"]]
# simplifing the actuals dataframe for comparison
actuals_conv_P_2019 = prices_US_conv_2019[["Date", "AveragePrice"]]
actuals_conv_P_2019.columns = ["ds", "y"]
# merging the prediction and actuals dataframes into one
conv_P_2019 = actuals_conv_P_2019.merge(pred_conv_P_2019, on="ds")
# plotting training, test and prediction data
conv_P_df.set_index("ds")["y"].plot(label="training")
conv_P_2019.set_index("ds")["y"].plot(label="test")
conv_P_2019.set_index("ds")["yhat"].plot(label="prediction")
plt.xlabel("Date")
plt.ylabel("Price")
plt.title("Price conventional - test vs. prediction")
plt.legend()
plt.savefig(str(constants.OUTPUTS_DIR) + "/Price conventional - test vs. prediction.png")
plt.show()
# model's MAPE calculation
model_conv_P_MAPE = MAPE(conv_P_2019["y"], conv_P_2019["yhat"])
# default vs. tuned models' MAPE
print("Default model MAPE: {:.2f}".format(model_conv_P_def_MAPE))
print("Tuned model MAPE: {:.2f}".format(model_conv_P_MAPE))
Default model MAPE: 18.27 Tuned model MAPE: 11.14
# simplifing the full dataset for use by Prophet
conv_P_full = prices_US_conv_merged[["Date", "AveragePrice"]]
conv_P_full.columns = ["ds", "y"]
# re-training the model using the best performing parameters and full dataset
# creating 6-month forecast
model_conv_P_for = Prophet(**best_params_conv_P)
model_conv_P_for.fit(conv_P_full)
future_6m_conv_P = model_conv_P_for.make_future_dataframe(periods=26, freq="W", include_history=False)
forecast_6m_conv_P = model_conv_P_for.predict(future_6m_conv_P)
fig_6m_conv_P = model_conv_P_for.plot(forecast_6m_conv_P)
# plotting forecast
conv_P_full.set_index("ds")["y"].plot(label="original")
forecast_6m_conv_P.set_index("ds")["yhat"].plot(label="forecast")
plt.xlabel("Date")
plt.ylabel("Price")
plt.title("Price conventional - forecast")
plt.legend()
plt.savefig(str(constants.OUTPUTS_DIR) + "/Price conventional - forecast.png")
plt.show()
# simplifing the dataframe for use by Prophet
org_P_df = prices_US_org_2015_2018[["Date", "AveragePrice"]]
org_P_df.columns = ["ds", "y"]
# creating model with default values, skipping only the daily and weekly seasonality which isn't present in our data
model_org_P_def = Prophet(daily_seasonality=False, weekly_seasonality=False)
# fitting the model on the whole training set
model_org_P_def.fit(org_P_df)
<prophet.forecaster.Prophet at 0x7fdcf82b3bd0>
future_org_P_def = model_org_P_def.make_future_dataframe(periods=len(prices_US_org_2019), freq="W")
# forecast visualisation
forecast_org_P_def = model_org_P_def.predict(future_org_P_def)
fig_org_P_def = model_org_P_def.plot(forecast_org_P_def)
# model components visualisation
fig_org_P_comp_def = model_org_P_def.plot_components(forecast_org_P_def)
# selecting only the test period forecast
pred_org_P_2019_def = forecast_org_P_def[forecast_org_P_def["ds"]>"2019-01-01"][["ds", "yhat"]]
# simplifing the actuals dataframe for comparison
actuals_org_P_2019_def = prices_US_org_2019[["Date", "AveragePrice"]]
actuals_org_P_2019_def.columns = ["ds", "y"]
# merging the prediction and actuals dataframes into one
org_P_2019_def = actuals_org_P_2019_def.merge(pred_org_P_2019_def, on="ds")
# plotting training, test and prediction data
org_P_df.set_index("ds")["y"].plot(label="training")
org_P_2019_def.set_index("ds")["y"].plot(label="test")
org_P_2019_def.set_index("ds")["yhat"].plot(label="prediction")
plt.xlabel("Date")
plt.ylabel("Price")
plt.title("Price organic - default model")
plt.legend()
plt.savefig(str(constants.OUTPUTS_DIR) + "/Price organic - default model.png")
plt.show()
# model's MAPE calculation
model_org_P_def_MAPE = MAPE(org_P_2019_def["y"], org_P_2019_def["yhat"])
print("Default model MAPE: {:.2f}".format(model_org_P_def_MAPE))
Default model MAPE: 19.01
# cross-validation hyperparameters on the Prophet model
param_grid = {
"changepoint_prior_scale": [0.001, 0.01, 0.1, 0.5],
"seasonality_prior_scale": [0.01, 0.1, 1.0, 10.0],
"seasonality_mode": ["additive", "multiplicative"],
"growth": ["linear", "flat"],
"daily_seasonality": [False],
"weekly_seasonality": [False]
}
# Generate all combinations of parameters
all_params = [dict(zip(param_grid.keys(), v)) for v in itertools.product(*param_grid.values())]
mapes = [] # Store the MAPEs for each params here
# Use cross validation to evaluate all parameters
for params in all_params:
m = Prophet(**params).fit(org_P_df) # Fit model with given params
df_cv_org_P = cross_validation(m, initial="730 days", period="90 days", horizon="365 days", parallel="processes")
df_p_org_P = performance_metrics(df_cv_org_P, rolling_window=1)
mapes.append(df_p_org_P["mape"].values[0])
# Find the best parameters
tuning_results = pd.DataFrame(all_params)
tuning_results["mape"] = mapes
print(tuning_results)
INFO:prophet:Making 4 forecasts with cutoffs between 2017-03-07 00:00:00 and 2017-12-02 00:00:00 INFO:prophet:Applying in parallel with <concurrent.futures.process.ProcessPoolExecutor object at 0x7fdcf624a1d0> WARNING:prophet.models:Optimization terminated abnormally. Falling back to Newton. WARNING:prophet.models:Optimization terminated abnormally. Falling back to Newton. WARNING:prophet.models:Optimization terminated abnormally. Falling back to Newton. INFO:prophet:Making 4 forecasts with cutoffs between 2017-03-07 00:00:00 and 2017-12-02 00:00:00 INFO:prophet:Applying in parallel with <concurrent.futures.process.ProcessPoolExecutor object at 0x7fdcd84ab650> INFO:prophet:Making 4 forecasts with cutoffs between 2017-03-07 00:00:00 and 2017-12-02 00:00:00 INFO:prophet:Applying in parallel with <concurrent.futures.process.ProcessPoolExecutor object at 0x7fdcf59fec50> WARNING:prophet.models:Optimization terminated abnormally. Falling back to Newton. WARNING:prophet.models:Optimization terminated abnormally. Falling back to Newton. WARNING:prophet.models:Optimization terminated abnormally. Falling back to Newton. INFO:prophet:Making 4 forecasts with cutoffs between 2017-03-07 00:00:00 and 2017-12-02 00:00:00 INFO:prophet:Applying in parallel with <concurrent.futures.process.ProcessPoolExecutor object at 0x7fdcf5314ad0> INFO:prophet:Making 4 forecasts with cutoffs between 2017-03-07 00:00:00 and 2017-12-02 00:00:00 INFO:prophet:Applying in parallel with <concurrent.futures.process.ProcessPoolExecutor object at 0x7fdcf5994fd0> WARNING:prophet.models:Optimization terminated abnormally. Falling back to Newton. WARNING:prophet.models:Optimization terminated abnormally. Falling back to Newton. WARNING:prophet.models:Optimization terminated abnormally. Falling back to Newton. INFO:prophet:Making 4 forecasts with cutoffs between 2017-03-07 00:00:00 and 2017-12-02 00:00:00 INFO:prophet:Applying in parallel with <concurrent.futures.process.ProcessPoolExecutor object at 0x7fdcf54a2190> INFO:prophet:Making 4 forecasts with cutoffs between 2017-03-07 00:00:00 and 2017-12-02 00:00:00 INFO:prophet:Applying in parallel with <concurrent.futures.process.ProcessPoolExecutor object at 0x7fdcf530f3d0> WARNING:prophet.models:Optimization terminated abnormally. Falling back to Newton. WARNING:prophet.models:Optimization terminated abnormally. Falling back to Newton. WARNING:prophet.models:Optimization terminated abnormally. Falling back to Newton. INFO:prophet:Making 4 forecasts with cutoffs between 2017-03-07 00:00:00 and 2017-12-02 00:00:00 INFO:prophet:Applying in parallel with <concurrent.futures.process.ProcessPoolExecutor object at 0x7fdcf586b090> INFO:prophet:Making 4 forecasts with cutoffs between 2017-03-07 00:00:00 and 2017-12-02 00:00:00 INFO:prophet:Applying in parallel with <concurrent.futures.process.ProcessPoolExecutor object at 0x7fdcf55d1e50> WARNING:prophet.models:Optimization terminated abnormally. Falling back to Newton. WARNING:prophet.models:Optimization terminated abnormally. Falling back to Newton. WARNING:prophet.models:Optimization terminated abnormally. Falling back to Newton. INFO:prophet:Making 4 forecasts with cutoffs between 2017-03-07 00:00:00 and 2017-12-02 00:00:00 INFO:prophet:Applying in parallel with <concurrent.futures.process.ProcessPoolExecutor object at 0x7fdcf5775d10> INFO:prophet:Making 4 forecasts with cutoffs between 2017-03-07 00:00:00 and 2017-12-02 00:00:00 INFO:prophet:Applying in parallel with <concurrent.futures.process.ProcessPoolExecutor object at 0x7fdcf576b810> WARNING:prophet.models:Optimization terminated abnormally. Falling back to Newton. WARNING:prophet.models:Optimization terminated abnormally. Falling back to Newton. WARNING:prophet.models:Optimization terminated abnormally. Falling back to Newton. INFO:prophet:Making 4 forecasts with cutoffs between 2017-03-07 00:00:00 and 2017-12-02 00:00:00 INFO:prophet:Applying in parallel with <concurrent.futures.process.ProcessPoolExecutor object at 0x7fdcf530fb90> INFO:prophet:Making 4 forecasts with cutoffs between 2017-03-07 00:00:00 and 2017-12-02 00:00:00 INFO:prophet:Applying in parallel with <concurrent.futures.process.ProcessPoolExecutor object at 0x7fdcf597e3d0> WARNING:prophet.models:Optimization terminated abnormally. Falling back to Newton. WARNING:prophet.models:Optimization terminated abnormally. Falling back to Newton. WARNING:prophet.models:Optimization terminated abnormally. Falling back to Newton. INFO:prophet:Making 4 forecasts with cutoffs between 2017-03-07 00:00:00 and 2017-12-02 00:00:00 INFO:prophet:Applying in parallel with <concurrent.futures.process.ProcessPoolExecutor object at 0x7fdcf576b810> INFO:prophet:Making 4 forecasts with cutoffs between 2017-03-07 00:00:00 and 2017-12-02 00:00:00 INFO:prophet:Applying in parallel with <concurrent.futures.process.ProcessPoolExecutor object at 0x7fdcf54a2190> WARNING:prophet.models:Optimization terminated abnormally. Falling back to Newton. WARNING:prophet.models:Optimization terminated abnormally. Falling back to Newton. WARNING:prophet.models:Optimization terminated abnormally. Falling back to Newton. INFO:prophet:Making 4 forecasts with cutoffs between 2017-03-07 00:00:00 and 2017-12-02 00:00:00 INFO:prophet:Applying in parallel with <concurrent.futures.process.ProcessPoolExecutor object at 0x7fdcf59f3910> INFO:prophet:Making 4 forecasts with cutoffs between 2017-03-07 00:00:00 and 2017-12-02 00:00:00 INFO:prophet:Applying in parallel with <concurrent.futures.process.ProcessPoolExecutor object at 0x7fdcf52fbb50> INFO:prophet:Making 4 forecasts with cutoffs between 2017-03-07 00:00:00 and 2017-12-02 00:00:00 INFO:prophet:Applying in parallel with <concurrent.futures.process.ProcessPoolExecutor object at 0x7fdcf5755350> INFO:prophet:Making 4 forecasts with cutoffs between 2017-03-07 00:00:00 and 2017-12-02 00:00:00 INFO:prophet:Applying in parallel with <concurrent.futures.process.ProcessPoolExecutor object at 0x7fdcf611c350> INFO:prophet:Making 4 forecasts with cutoffs between 2017-03-07 00:00:00 and 2017-12-02 00:00:00 INFO:prophet:Applying in parallel with <concurrent.futures.process.ProcessPoolExecutor object at 0x7fdcf5820410> INFO:prophet:Making 4 forecasts with cutoffs between 2017-03-07 00:00:00 and 2017-12-02 00:00:00 INFO:prophet:Applying in parallel with <concurrent.futures.process.ProcessPoolExecutor object at 0x7fdcf61b5210> INFO:prophet:Making 4 forecasts with cutoffs between 2017-03-07 00:00:00 and 2017-12-02 00:00:00 INFO:prophet:Applying in parallel with <concurrent.futures.process.ProcessPoolExecutor object at 0x7fdcf58b5090> INFO:prophet:Making 4 forecasts with cutoffs between 2017-03-07 00:00:00 and 2017-12-02 00:00:00 INFO:prophet:Applying in parallel with <concurrent.futures.process.ProcessPoolExecutor object at 0x7fdcf5849490> INFO:prophet:Making 4 forecasts with cutoffs between 2017-03-07 00:00:00 and 2017-12-02 00:00:00 INFO:prophet:Applying in parallel with <concurrent.futures.process.ProcessPoolExecutor object at 0x7fdcf629ea50> INFO:prophet:Making 4 forecasts with cutoffs between 2017-03-07 00:00:00 and 2017-12-02 00:00:00 INFO:prophet:Applying in parallel with <concurrent.futures.process.ProcessPoolExecutor object at 0x7fdcf589f050> INFO:prophet:Making 4 forecasts with cutoffs between 2017-03-07 00:00:00 and 2017-12-02 00:00:00 INFO:prophet:Applying in parallel with <concurrent.futures.process.ProcessPoolExecutor object at 0x7fdcf5894fd0> INFO:prophet:Making 4 forecasts with cutoffs between 2017-03-07 00:00:00 and 2017-12-02 00:00:00 INFO:prophet:Applying in parallel with <concurrent.futures.process.ProcessPoolExecutor object at 0x7fdcf82366d0> INFO:prophet:Making 4 forecasts with cutoffs between 2017-03-07 00:00:00 and 2017-12-02 00:00:00 INFO:prophet:Applying in parallel with <concurrent.futures.process.ProcessPoolExecutor object at 0x7fdcf61acb90> INFO:prophet:Making 4 forecasts with cutoffs between 2017-03-07 00:00:00 and 2017-12-02 00:00:00 INFO:prophet:Applying in parallel with <concurrent.futures.process.ProcessPoolExecutor object at 0x7fdcf59fec50> INFO:prophet:Making 4 forecasts with cutoffs between 2017-03-07 00:00:00 and 2017-12-02 00:00:00 INFO:prophet:Applying in parallel with <concurrent.futures.process.ProcessPoolExecutor object at 0x7fdcf5ccfe10> INFO:prophet:Making 4 forecasts with cutoffs between 2017-03-07 00:00:00 and 2017-12-02 00:00:00 INFO:prophet:Applying in parallel with <concurrent.futures.process.ProcessPoolExecutor object at 0x7fdcf5a090d0> INFO:prophet:Making 4 forecasts with cutoffs between 2017-03-07 00:00:00 and 2017-12-02 00:00:00 INFO:prophet:Applying in parallel with <concurrent.futures.process.ProcessPoolExecutor object at 0x7fdcf5ccf1d0> INFO:prophet:Making 4 forecasts with cutoffs between 2017-03-07 00:00:00 and 2017-12-02 00:00:00 INFO:prophet:Applying in parallel with <concurrent.futures.process.ProcessPoolExecutor object at 0x7fdcf6994050> INFO:prophet:Making 4 forecasts with cutoffs between 2017-03-07 00:00:00 and 2017-12-02 00:00:00 INFO:prophet:Applying in parallel with <concurrent.futures.process.ProcessPoolExecutor object at 0x7fdcd849b590> INFO:prophet:Making 4 forecasts with cutoffs between 2017-03-07 00:00:00 and 2017-12-02 00:00:00 INFO:prophet:Applying in parallel with <concurrent.futures.process.ProcessPoolExecutor object at 0x7fdcf7a99710> INFO:prophet:Making 4 forecasts with cutoffs between 2017-03-07 00:00:00 and 2017-12-02 00:00:00 INFO:prophet:Applying in parallel with <concurrent.futures.process.ProcessPoolExecutor object at 0x7fdcf791d450> INFO:prophet:Making 4 forecasts with cutoffs between 2017-03-07 00:00:00 and 2017-12-02 00:00:00 INFO:prophet:Applying in parallel with <concurrent.futures.process.ProcessPoolExecutor object at 0x7fdcf8267fd0> INFO:prophet:Making 4 forecasts with cutoffs between 2017-03-07 00:00:00 and 2017-12-02 00:00:00 INFO:prophet:Applying in parallel with <concurrent.futures.process.ProcessPoolExecutor object at 0x7fdcf82c2b10> INFO:prophet:Making 4 forecasts with cutoffs between 2017-03-07 00:00:00 and 2017-12-02 00:00:00 INFO:prophet:Applying in parallel with <concurrent.futures.process.ProcessPoolExecutor object at 0x7fdcf58154d0> INFO:prophet:Making 4 forecasts with cutoffs between 2017-03-07 00:00:00 and 2017-12-02 00:00:00 INFO:prophet:Applying in parallel with <concurrent.futures.process.ProcessPoolExecutor object at 0x7fdcf62cd610> INFO:prophet:Making 4 forecasts with cutoffs between 2017-03-07 00:00:00 and 2017-12-02 00:00:00 INFO:prophet:Applying in parallel with <concurrent.futures.process.ProcessPoolExecutor object at 0x7fdcf61acb90> INFO:prophet:Making 4 forecasts with cutoffs between 2017-03-07 00:00:00 and 2017-12-02 00:00:00 INFO:prophet:Applying in parallel with <concurrent.futures.process.ProcessPoolExecutor object at 0x7fdcf7ad5390> INFO:prophet:Making 4 forecasts with cutoffs between 2017-03-07 00:00:00 and 2017-12-02 00:00:00 INFO:prophet:Applying in parallel with <concurrent.futures.process.ProcessPoolExecutor object at 0x7fdcf6297a50> INFO:prophet:Making 4 forecasts with cutoffs between 2017-03-07 00:00:00 and 2017-12-02 00:00:00 INFO:prophet:Applying in parallel with <concurrent.futures.process.ProcessPoolExecutor object at 0x7fdcf7a0bf10> INFO:prophet:Making 4 forecasts with cutoffs between 2017-03-07 00:00:00 and 2017-12-02 00:00:00 INFO:prophet:Applying in parallel with <concurrent.futures.process.ProcessPoolExecutor object at 0x7fdcf587f050> INFO:prophet:Making 4 forecasts with cutoffs between 2017-03-07 00:00:00 and 2017-12-02 00:00:00 INFO:prophet:Applying in parallel with <concurrent.futures.process.ProcessPoolExecutor object at 0x7fdcf6a91890> INFO:prophet:Making 4 forecasts with cutoffs between 2017-03-07 00:00:00 and 2017-12-02 00:00:00 INFO:prophet:Applying in parallel with <concurrent.futures.process.ProcessPoolExecutor object at 0x7fdcf69a9a10> INFO:prophet:Making 4 forecasts with cutoffs between 2017-03-07 00:00:00 and 2017-12-02 00:00:00 INFO:prophet:Applying in parallel with <concurrent.futures.process.ProcessPoolExecutor object at 0x7fdcf7ae5050> INFO:prophet:Making 4 forecasts with cutoffs between 2017-03-07 00:00:00 and 2017-12-02 00:00:00 INFO:prophet:Applying in parallel with <concurrent.futures.process.ProcessPoolExecutor object at 0x7fdcf7c4b350> INFO:prophet:Making 4 forecasts with cutoffs between 2017-03-07 00:00:00 and 2017-12-02 00:00:00 INFO:prophet:Applying in parallel with <concurrent.futures.process.ProcessPoolExecutor object at 0x7fdcf6297a50> INFO:prophet:Making 4 forecasts with cutoffs between 2017-03-07 00:00:00 and 2017-12-02 00:00:00 INFO:prophet:Applying in parallel with <concurrent.futures.process.ProcessPoolExecutor object at 0x7fdcf7c4b2d0> INFO:prophet:Making 4 forecasts with cutoffs between 2017-03-07 00:00:00 and 2017-12-02 00:00:00 INFO:prophet:Applying in parallel with <concurrent.futures.process.ProcessPoolExecutor object at 0x7fdcf5953fd0> INFO:prophet:Making 4 forecasts with cutoffs between 2017-03-07 00:00:00 and 2017-12-02 00:00:00 INFO:prophet:Applying in parallel with <concurrent.futures.process.ProcessPoolExecutor object at 0x7fdcf7a99b90> INFO:prophet:Making 4 forecasts with cutoffs between 2017-03-07 00:00:00 and 2017-12-02 00:00:00 INFO:prophet:Applying in parallel with <concurrent.futures.process.ProcessPoolExecutor object at 0x7fdcf6f33b10> INFO:prophet:Making 4 forecasts with cutoffs between 2017-03-07 00:00:00 and 2017-12-02 00:00:00 INFO:prophet:Applying in parallel with <concurrent.futures.process.ProcessPoolExecutor object at 0x7fdcf618fed0> INFO:prophet:Making 4 forecasts with cutoffs between 2017-03-07 00:00:00 and 2017-12-02 00:00:00 INFO:prophet:Applying in parallel with <concurrent.futures.process.ProcessPoolExecutor object at 0x7fdcf82102d0> INFO:prophet:Making 4 forecasts with cutoffs between 2017-03-07 00:00:00 and 2017-12-02 00:00:00 INFO:prophet:Applying in parallel with <concurrent.futures.process.ProcessPoolExecutor object at 0x7fdcf618ff10> INFO:prophet:Making 4 forecasts with cutoffs between 2017-03-07 00:00:00 and 2017-12-02 00:00:00 INFO:prophet:Applying in parallel with <concurrent.futures.process.ProcessPoolExecutor object at 0x7fdcf7a99b90> INFO:prophet:Making 4 forecasts with cutoffs between 2017-03-07 00:00:00 and 2017-12-02 00:00:00 INFO:prophet:Applying in parallel with <concurrent.futures.process.ProcessPoolExecutor object at 0x7fdcd84b4450> INFO:prophet:Making 4 forecasts with cutoffs between 2017-03-07 00:00:00 and 2017-12-02 00:00:00 INFO:prophet:Applying in parallel with <concurrent.futures.process.ProcessPoolExecutor object at 0x7fdcf7ae5990> INFO:prophet:Making 4 forecasts with cutoffs between 2017-03-07 00:00:00 and 2017-12-02 00:00:00 INFO:prophet:Applying in parallel with <concurrent.futures.process.ProcessPoolExecutor object at 0x7fdcf6f33b10> INFO:prophet:Making 4 forecasts with cutoffs between 2017-03-07 00:00:00 and 2017-12-02 00:00:00 INFO:prophet:Applying in parallel with <concurrent.futures.process.ProcessPoolExecutor object at 0x7fdcf7939050> INFO:prophet:Making 4 forecasts with cutoffs between 2017-03-07 00:00:00 and 2017-12-02 00:00:00 INFO:prophet:Applying in parallel with <concurrent.futures.process.ProcessPoolExecutor object at 0x7fdcf81f9fd0> INFO:prophet:Making 4 forecasts with cutoffs between 2017-03-07 00:00:00 and 2017-12-02 00:00:00 INFO:prophet:Applying in parallel with <concurrent.futures.process.ProcessPoolExecutor object at 0x7fdcf6f33b10>
changepoint_prior_scale seasonality_prior_scale seasonality_mode growth \
0 0.00 0.01 additive linear
1 0.00 0.01 additive flat
2 0.00 0.01 multiplicative linear
3 0.00 0.01 multiplicative flat
4 0.00 0.10 additive linear
.. ... ... ... ...
59 0.50 1.00 multiplicative flat
60 0.50 10.00 additive linear
61 0.50 10.00 additive flat
62 0.50 10.00 multiplicative linear
63 0.50 10.00 multiplicative flat
daily_seasonality weekly_seasonality mape
0 False False 0.10
1 False False 0.07
2 False False 0.11
3 False False 0.07
4 False False 0.11
.. ... ... ...
59 False False 0.07
60 False False 0.25
61 False False 0.07
62 False False 0.24
63 False False 0.07
[64 rows x 7 columns]
# selecting the best hyperparameters
best_params_org_P = all_params[np.argmin(mapes)]
print(best_params_org_P)
{'changepoint_prior_scale': 0.001, 'seasonality_prior_scale': 0.01, 'seasonality_mode': 'additive', 'growth': 'flat', 'daily_seasonality': False, 'weekly_seasonality': False}
# creating the model with the best parameters
model_org_P = Prophet(**best_params_org_P)
# fitting the model on the whole training set
model_org_P.fit(org_P_df)
<prophet.forecaster.Prophet at 0x7fdcf59df610>
len(prices_US_org_2019)
28
future_org_P = model_org_P.make_future_dataframe(periods=len(prices_US_org_2019), freq="W")
# forecast visualisation
forecast_org_P = model_org_P.predict(future_org_P)
fig_org_P = model_org_P.plot(forecast_org_P)
# model components visualisation
fig_org_P_comp = model_org_P.plot_components(forecast_org_P)
# cross-validation of the selected model
df_cv_org_P = cross_validation(model_org_P, initial="730 days", period="90 days", horizon="365 days", parallel="processes")
INFO:prophet:Making 4 forecasts with cutoffs between 2017-03-07 00:00:00 and 2017-12-02 00:00:00 INFO:prophet:Applying in parallel with <concurrent.futures.process.ProcessPoolExecutor object at 0x7fdcf6189e50>
# MAPE of the cross-validated model visualisation
fig_org_P_cv_mape = plot_cross_validation_metric(df_cv_org_P, metric='mape')
# selecting only the test period forecast
pred_org_P_2019 = forecast_org_P[forecast_org_P["ds"]>"2019-01-01"][["ds", "yhat"]]
# simplifing the actuals dataframe for comparison
actuals_org_P_2019 = prices_US_org_2019[["Date", "AveragePrice"]]
actuals_org_P_2019.columns = ["ds", "y"]
# merging the prediction and actuals dataframes into one
org_P_2019 = actuals_org_P_2019.merge(pred_org_P_2019, on="ds")
# plotting training, test and prediction data
org_P_df.set_index("ds")["y"].plot(label="training")
org_P_2019.set_index("ds")["y"].plot(label="test")
org_P_2019.set_index("ds")["yhat"].plot(label="prediction")
plt.xlabel("Date")
plt.ylabel("Price")
plt.title("Price organic - test vs. prediction")
plt.legend()
plt.savefig(str(constants.OUTPUTS_DIR) + "/Price organic - test vs. prediction.png")
plt.show()
# model's MAPE calculation
model_org_P_MAPE = MAPE(org_P_2019["y"], org_P_2019["yhat"])
# default vs. tuned models' MAPE
print("Default model MAPE: {:.2f}".format(model_org_P_def_MAPE))
print("Tuned model MAPE: {:.2f}".format(model_org_P_MAPE))
Default model MAPE: 19.01 Tuned model MAPE: 8.35
# simplifing the full dataset for use by Prophet
org_P_full = prices_US_org_merged[["Date", "AveragePrice"]]
org_P_full.columns = ["ds", "y"]
# re-training the model using the best performing parameters and full dataset
# creating 6-month (= 26-week) forecast
model_org_P_for = Prophet(**best_params_org_P)
model_org_P_for.fit(org_P_full)
future_6m_org_P = model_org_P_for.make_future_dataframe(periods=26, freq="W", include_history=False)
forecast_6m_org_P = model_org_P_for.predict(future_6m_org_P)
fig_6m_org_P = model_org_P_for.plot(forecast_6m_org_P)
# plotting forecast
org_P_full.set_index("ds")["y"].plot(label="original")
forecast_6m_org_P.set_index("ds")["yhat"].plot(label="forecast")
plt.xlabel("Date")
plt.ylabel("Price")
plt.title("Price organic - forecast")
plt.legend()
plt.savefig(str(constants.OUTPUTS_DIR) + "/Price organic - forecast.png")
plt.show()
# simplifing the dataframe for use by Prophet
conv_Q_df = prices_US_conv_2015_2018[["Date", "TotalVolume"]]
conv_Q_df.columns = ["ds", "y"]
# conv_Q_df["cap"]
# creating model with default values, skipping only the daily and weekly seasonality which isn't present in our data
model_conv_Q_def = Prophet(daily_seasonality=False, weekly_seasonality=False)
# fitting the model on the whole training set
model_conv_Q_def.fit(conv_Q_df)
<prophet.forecaster.Prophet at 0x7fdcf7a99ed0>
future_conv_Q_def = model_conv_Q_def.make_future_dataframe(periods=len(prices_US_conv_2019), freq="W")
# forecast visualisation
forecast_conv_Q_def = model_conv_Q_def.predict(future_conv_Q_def)
fig_conv_Q_def = model_conv_Q_def.plot(forecast_conv_Q_def)
# model components visualisation
fig_conv_Q_comp_def = model_conv_Q_def.plot_components(forecast_conv_Q_def)
# selecting only the test period forecast
pred_conv_Q_2019_def = forecast_conv_Q_def[forecast_conv_Q_def["ds"]>"2019-01-01"][["ds", "yhat"]]
# simplifing the actuals dataframe for comparison
actuals_conv_Q_2019_def = prices_US_conv_2019[["Date", "TotalVolume"]]
actuals_conv_Q_2019_def.columns = ["ds", "y"]
# merging the prediction and actuals dataframes into one
conv_Q_2019_def = actuals_conv_Q_2019_def.merge(pred_conv_Q_2019_def, on="ds")
# plotting training, test and prediction data
conv_Q_df.set_index("ds")["y"].plot(label="training")
conv_Q_2019_def.set_index("ds")["y"].plot(label="test")
conv_Q_2019_def.set_index("ds")["yhat"].plot(label="prediction")
plt.xlabel("Date")
plt.ylabel("Volume")
plt.title("Volume conventional - default model")
plt.legend()
plt.savefig(str(constants.OUTPUTS_DIR) + "/Volume conventional - default model.png")
plt.show()
# model's MAPE calculation
model_conv_Q_def_MAPE = MAPE(conv_Q_2019_def["y"], conv_Q_2019_def["yhat"])
print("Default model MAPE: {:.2f}".format(model_conv_Q_def_MAPE))
Default model MAPE: 13.46
# cross-validation hyperparameters on the Prophet model
param_grid = {
"changepoint_prior_scale": [0.001, 0.01, 0.1, 0.5],
"seasonality_prior_scale": [0.01, 0.1, 1.0, 10.0],
"growth": ["linear", "flat"], # "logistic" might be better for this model but we'd need the cap value estimate from the client
"seasonality_mode": ["additive", "multiplicative"],
"daily_seasonality": [False],
"weekly_seasonality": [False]
}
# Generate all combinations of parameters
all_params = [dict(zip(param_grid.keys(), v)) for v in itertools.product(*param_grid.values())]
mapes = [] # Store the MAPEs for each params here
# Use cross validation to evaluate all parameters
for params in all_params:
m = Prophet(**params).fit(conv_Q_df) # Fit model with given params
df_cv_conv_Q = cross_validation(m, initial="730 days", period="90 days", horizon="365 days", parallel="processes")
df_p_conv_Q = performance_metrics(df_cv_conv_Q, rolling_window=1)
mapes.append(df_p_conv_Q["mape"].values[0])
# Find the best parameters
tuning_results = pd.DataFrame(all_params)
tuning_results["mape"] = mapes
print(tuning_results)
WARNING:prophet.models:Optimization terminated abnormally. Falling back to Newton. INFO:prophet:Making 4 forecasts with cutoffs between 2017-03-07 00:00:00 and 2017-12-02 00:00:00 INFO:prophet:Applying in parallel with <concurrent.futures.process.ProcessPoolExecutor object at 0x7fdcf7ae46d0> WARNING:prophet.models:Optimization terminated abnormally. Falling back to Newton. WARNING:prophet.models:Optimization terminated abnormally. Falling back to Newton. WARNING:prophet.models:Optimization terminated abnormally. Falling back to Newton. WARNING:prophet.models:Optimization terminated abnormally. Falling back to Newton. INFO:prophet:Making 4 forecasts with cutoffs between 2017-03-07 00:00:00 and 2017-12-02 00:00:00 INFO:prophet:Applying in parallel with <concurrent.futures.process.ProcessPoolExecutor object at 0x7fdcd8e46110> WARNING:prophet.models:Optimization terminated abnormally. Falling back to Newton. WARNING:prophet.models:Optimization terminated abnormally. Falling back to Newton. WARNING:prophet.models:Optimization terminated abnormally. Falling back to Newton. INFO:prophet:Making 4 forecasts with cutoffs between 2017-03-07 00:00:00 and 2017-12-02 00:00:00 INFO:prophet:Applying in parallel with <concurrent.futures.process.ProcessPoolExecutor object at 0x7fdcd97418d0> INFO:prophet:Making 4 forecasts with cutoffs between 2017-03-07 00:00:00 and 2017-12-02 00:00:00 INFO:prophet:Applying in parallel with <concurrent.futures.process.ProcessPoolExecutor object at 0x7fdcf5800050> WARNING:prophet.models:Optimization terminated abnormally. Falling back to Newton. INFO:prophet:Making 4 forecasts with cutoffs between 2017-03-07 00:00:00 and 2017-12-02 00:00:00 INFO:prophet:Applying in parallel with <concurrent.futures.process.ProcessPoolExecutor object at 0x7fdcda26bd10> WARNING:prophet.models:Optimization terminated abnormally. Falling back to Newton. WARNING:prophet.models:Optimization terminated abnormally. Falling back to Newton. WARNING:prophet.models:Optimization terminated abnormally. Falling back to Newton. WARNING:prophet.models:Optimization terminated abnormally. Falling back to Newton. INFO:prophet:Making 4 forecasts with cutoffs between 2017-03-07 00:00:00 and 2017-12-02 00:00:00 INFO:prophet:Applying in parallel with <concurrent.futures.process.ProcessPoolExecutor object at 0x7fdcda2236d0> WARNING:prophet.models:Optimization terminated abnormally. Falling back to Newton. WARNING:prophet.models:Optimization terminated abnormally. Falling back to Newton. WARNING:prophet.models:Optimization terminated abnormally. Falling back to Newton. INFO:prophet:Making 4 forecasts with cutoffs between 2017-03-07 00:00:00 and 2017-12-02 00:00:00 INFO:prophet:Applying in parallel with <concurrent.futures.process.ProcessPoolExecutor object at 0x7fdcd8e46f50> INFO:prophet:Making 4 forecasts with cutoffs between 2017-03-07 00:00:00 and 2017-12-02 00:00:00 INFO:prophet:Applying in parallel with <concurrent.futures.process.ProcessPoolExecutor object at 0x7fdcf69cfc50> WARNING:prophet.models:Optimization terminated abnormally. Falling back to Newton. INFO:prophet:Making 4 forecasts with cutoffs between 2017-03-07 00:00:00 and 2017-12-02 00:00:00 INFO:prophet:Applying in parallel with <concurrent.futures.process.ProcessPoolExecutor object at 0x7fdcf69ed610> WARNING:prophet.models:Optimization terminated abnormally. Falling back to Newton. WARNING:prophet.models:Optimization terminated abnormally. Falling back to Newton. WARNING:prophet.models:Optimization terminated abnormally. Falling back to Newton. WARNING:prophet.models:Optimization terminated abnormally. Falling back to Newton. INFO:prophet:Making 4 forecasts with cutoffs between 2017-03-07 00:00:00 and 2017-12-02 00:00:00 INFO:prophet:Applying in parallel with <concurrent.futures.process.ProcessPoolExecutor object at 0x7fdcd8e46f50> WARNING:prophet.models:Optimization terminated abnormally. Falling back to Newton. WARNING:prophet.models:Optimization terminated abnormally. Falling back to Newton. WARNING:prophet.models:Optimization terminated abnormally. Falling back to Newton. INFO:prophet:Making 4 forecasts with cutoffs between 2017-03-07 00:00:00 and 2017-12-02 00:00:00 INFO:prophet:Applying in parallel with <concurrent.futures.process.ProcessPoolExecutor object at 0x7fdcf82ad9d0> INFO:prophet:Making 4 forecasts with cutoffs between 2017-03-07 00:00:00 and 2017-12-02 00:00:00 INFO:prophet:Applying in parallel with <concurrent.futures.process.ProcessPoolExecutor object at 0x7fdcf57e1790> WARNING:prophet.models:Optimization terminated abnormally. Falling back to Newton. INFO:prophet:Making 4 forecasts with cutoffs between 2017-03-07 00:00:00 and 2017-12-02 00:00:00 INFO:prophet:Applying in parallel with <concurrent.futures.process.ProcessPoolExecutor object at 0x7fdcf6fd1e90> WARNING:prophet.models:Optimization terminated abnormally. Falling back to Newton. WARNING:prophet.models:Optimization terminated abnormally. Falling back to Newton. WARNING:prophet.models:Optimization terminated abnormally. Falling back to Newton. WARNING:prophet.models:Optimization terminated abnormally. Falling back to Newton. INFO:prophet:Making 4 forecasts with cutoffs between 2017-03-07 00:00:00 and 2017-12-02 00:00:00 INFO:prophet:Applying in parallel with <concurrent.futures.process.ProcessPoolExecutor object at 0x7fdcda210b90> WARNING:prophet.models:Optimization terminated abnormally. Falling back to Newton. WARNING:prophet.models:Optimization terminated abnormally. Falling back to Newton. WARNING:prophet.models:Optimization terminated abnormally. Falling back to Newton. INFO:prophet:Making 4 forecasts with cutoffs between 2017-03-07 00:00:00 and 2017-12-02 00:00:00 INFO:prophet:Applying in parallel with <concurrent.futures.process.ProcessPoolExecutor object at 0x7fdcda223d90> INFO:prophet:Making 4 forecasts with cutoffs between 2017-03-07 00:00:00 and 2017-12-02 00:00:00 INFO:prophet:Applying in parallel with <concurrent.futures.process.ProcessPoolExecutor object at 0x7fdcf57eb590> INFO:prophet:Making 4 forecasts with cutoffs between 2017-03-07 00:00:00 and 2017-12-02 00:00:00 INFO:prophet:Applying in parallel with <concurrent.futures.process.ProcessPoolExecutor object at 0x7fdcd90c0a10> INFO:prophet:Making 4 forecasts with cutoffs between 2017-03-07 00:00:00 and 2017-12-02 00:00:00 INFO:prophet:Applying in parallel with <concurrent.futures.process.ProcessPoolExecutor object at 0x7fdcf7af5a10> INFO:prophet:Making 4 forecasts with cutoffs between 2017-03-07 00:00:00 and 2017-12-02 00:00:00 INFO:prophet:Applying in parallel with <concurrent.futures.process.ProcessPoolExecutor object at 0x7fdcf6a91950> INFO:prophet:Making 4 forecasts with cutoffs between 2017-03-07 00:00:00 and 2017-12-02 00:00:00 INFO:prophet:Applying in parallel with <concurrent.futures.process.ProcessPoolExecutor object at 0x7fdcf57e17d0> INFO:prophet:Making 4 forecasts with cutoffs between 2017-03-07 00:00:00 and 2017-12-02 00:00:00 INFO:prophet:Applying in parallel with <concurrent.futures.process.ProcessPoolExecutor object at 0x7fdcda2235d0> INFO:prophet:Making 4 forecasts with cutoffs between 2017-03-07 00:00:00 and 2017-12-02 00:00:00 INFO:prophet:Applying in parallel with <concurrent.futures.process.ProcessPoolExecutor object at 0x7fdcda210490> INFO:prophet:Making 4 forecasts with cutoffs between 2017-03-07 00:00:00 and 2017-12-02 00:00:00 INFO:prophet:Applying in parallel with <concurrent.futures.process.ProcessPoolExecutor object at 0x7fdcf6a02810> INFO:prophet:Making 4 forecasts with cutoffs between 2017-03-07 00:00:00 and 2017-12-02 00:00:00 INFO:prophet:Applying in parallel with <concurrent.futures.process.ProcessPoolExecutor object at 0x7fdcd8e46950> INFO:prophet:Making 4 forecasts with cutoffs between 2017-03-07 00:00:00 and 2017-12-02 00:00:00 INFO:prophet:Applying in parallel with <concurrent.futures.process.ProcessPoolExecutor object at 0x7fdcda231410> INFO:prophet:Making 4 forecasts with cutoffs between 2017-03-07 00:00:00 and 2017-12-02 00:00:00 INFO:prophet:Applying in parallel with <concurrent.futures.process.ProcessPoolExecutor object at 0x7fdcf7939050> INFO:prophet:Making 4 forecasts with cutoffs between 2017-03-07 00:00:00 and 2017-12-02 00:00:00 INFO:prophet:Applying in parallel with <concurrent.futures.process.ProcessPoolExecutor object at 0x7fdcf7af5a10> INFO:prophet:Making 4 forecasts with cutoffs between 2017-03-07 00:00:00 and 2017-12-02 00:00:00 INFO:prophet:Applying in parallel with <concurrent.futures.process.ProcessPoolExecutor object at 0x7fdcf7ae5050> INFO:prophet:Making 4 forecasts with cutoffs between 2017-03-07 00:00:00 and 2017-12-02 00:00:00 INFO:prophet:Applying in parallel with <concurrent.futures.process.ProcessPoolExecutor object at 0x7fdcda25f9d0> INFO:prophet:Making 4 forecasts with cutoffs between 2017-03-07 00:00:00 and 2017-12-02 00:00:00 INFO:prophet:Applying in parallel with <concurrent.futures.process.ProcessPoolExecutor object at 0x7fdcda25f150> INFO:prophet:Making 4 forecasts with cutoffs between 2017-03-07 00:00:00 and 2017-12-02 00:00:00 INFO:prophet:Applying in parallel with <concurrent.futures.process.ProcessPoolExecutor object at 0x7fdcf824efd0> INFO:prophet:Making 4 forecasts with cutoffs between 2017-03-07 00:00:00 and 2017-12-02 00:00:00 INFO:prophet:Applying in parallel with <concurrent.futures.process.ProcessPoolExecutor object at 0x7fdcf7ae0d90> INFO:prophet:Making 4 forecasts with cutoffs between 2017-03-07 00:00:00 and 2017-12-02 00:00:00 INFO:prophet:Applying in parallel with <concurrent.futures.process.ProcessPoolExecutor object at 0x7fdcf69cfc50> INFO:prophet:Making 4 forecasts with cutoffs between 2017-03-07 00:00:00 and 2017-12-02 00:00:00 INFO:prophet:Applying in parallel with <concurrent.futures.process.ProcessPoolExecutor object at 0x7fdcf6fd1e90> INFO:prophet:Making 4 forecasts with cutoffs between 2017-03-07 00:00:00 and 2017-12-02 00:00:00 INFO:prophet:Applying in parallel with <concurrent.futures.process.ProcessPoolExecutor object at 0x7fdcf61895d0> INFO:prophet:Making 4 forecasts with cutoffs between 2017-03-07 00:00:00 and 2017-12-02 00:00:00 INFO:prophet:Applying in parallel with <concurrent.futures.process.ProcessPoolExecutor object at 0x7fdcf57eb590> INFO:prophet:Making 4 forecasts with cutoffs between 2017-03-07 00:00:00 and 2017-12-02 00:00:00 INFO:prophet:Applying in parallel with <concurrent.futures.process.ProcessPoolExecutor object at 0x7fdcf82adb90> INFO:prophet:Making 4 forecasts with cutoffs between 2017-03-07 00:00:00 and 2017-12-02 00:00:00 INFO:prophet:Applying in parallel with <concurrent.futures.process.ProcessPoolExecutor object at 0x7fdcf69ed8d0> INFO:prophet:Making 4 forecasts with cutoffs between 2017-03-07 00:00:00 and 2017-12-02 00:00:00 INFO:prophet:Applying in parallel with <concurrent.futures.process.ProcessPoolExecutor object at 0x7fdcf7af5a10> INFO:prophet:Making 4 forecasts with cutoffs between 2017-03-07 00:00:00 and 2017-12-02 00:00:00 INFO:prophet:Applying in parallel with <concurrent.futures.process.ProcessPoolExecutor object at 0x7fdcf6fd1e90> INFO:prophet:Making 4 forecasts with cutoffs between 2017-03-07 00:00:00 and 2017-12-02 00:00:00 INFO:prophet:Applying in parallel with <concurrent.futures.process.ProcessPoolExecutor object at 0x7fdcda26bcd0> INFO:prophet:Making 4 forecasts with cutoffs between 2017-03-07 00:00:00 and 2017-12-02 00:00:00 INFO:prophet:Applying in parallel with <concurrent.futures.process.ProcessPoolExecutor object at 0x7fdcf7ae8c50> INFO:prophet:Making 4 forecasts with cutoffs between 2017-03-07 00:00:00 and 2017-12-02 00:00:00 INFO:prophet:Applying in parallel with <concurrent.futures.process.ProcessPoolExecutor object at 0x7fdcda223d90> INFO:prophet:Making 4 forecasts with cutoffs between 2017-03-07 00:00:00 and 2017-12-02 00:00:00 INFO:prophet:Applying in parallel with <concurrent.futures.process.ProcessPoolExecutor object at 0x7fdcf57e17d0> INFO:prophet:Making 4 forecasts with cutoffs between 2017-03-07 00:00:00 and 2017-12-02 00:00:00 INFO:prophet:Applying in parallel with <concurrent.futures.process.ProcessPoolExecutor object at 0x7fdcf82adb90> INFO:prophet:Making 4 forecasts with cutoffs between 2017-03-07 00:00:00 and 2017-12-02 00:00:00 INFO:prophet:Applying in parallel with <concurrent.futures.process.ProcessPoolExecutor object at 0x7fdcf69cfb50> INFO:prophet:Making 4 forecasts with cutoffs between 2017-03-07 00:00:00 and 2017-12-02 00:00:00 INFO:prophet:Applying in parallel with <concurrent.futures.process.ProcessPoolExecutor object at 0x7fdcf82adb90> INFO:prophet:Making 4 forecasts with cutoffs between 2017-03-07 00:00:00 and 2017-12-02 00:00:00 INFO:prophet:Applying in parallel with <concurrent.futures.process.ProcessPoolExecutor object at 0x7fdcf7ae8c50> INFO:prophet:Making 4 forecasts with cutoffs between 2017-03-07 00:00:00 and 2017-12-02 00:00:00 INFO:prophet:Applying in parallel with <concurrent.futures.process.ProcessPoolExecutor object at 0x7fdcda210b90> INFO:prophet:Making 4 forecasts with cutoffs between 2017-03-07 00:00:00 and 2017-12-02 00:00:00 INFO:prophet:Applying in parallel with <concurrent.futures.process.ProcessPoolExecutor object at 0x7fdcda27b9d0> INFO:prophet:Making 4 forecasts with cutoffs between 2017-03-07 00:00:00 and 2017-12-02 00:00:00 INFO:prophet:Applying in parallel with <concurrent.futures.process.ProcessPoolExecutor object at 0x7fdcda231410> INFO:prophet:Making 4 forecasts with cutoffs between 2017-03-07 00:00:00 and 2017-12-02 00:00:00 INFO:prophet:Applying in parallel with <concurrent.futures.process.ProcessPoolExecutor object at 0x7fdcda2ad790> INFO:prophet:Making 4 forecasts with cutoffs between 2017-03-07 00:00:00 and 2017-12-02 00:00:00 INFO:prophet:Applying in parallel with <concurrent.futures.process.ProcessPoolExecutor object at 0x7fdcf8210b90> INFO:prophet:Making 4 forecasts with cutoffs between 2017-03-07 00:00:00 and 2017-12-02 00:00:00 INFO:prophet:Applying in parallel with <concurrent.futures.process.ProcessPoolExecutor object at 0x7fdcf7ae4390> INFO:prophet:Making 4 forecasts with cutoffs between 2017-03-07 00:00:00 and 2017-12-02 00:00:00 INFO:prophet:Applying in parallel with <concurrent.futures.process.ProcessPoolExecutor object at 0x7fdcd84b4490> INFO:prophet:Making 4 forecasts with cutoffs between 2017-03-07 00:00:00 and 2017-12-02 00:00:00 INFO:prophet:Applying in parallel with <concurrent.futures.process.ProcessPoolExecutor object at 0x7fdcf69cfc50> INFO:prophet:Making 4 forecasts with cutoffs between 2017-03-07 00:00:00 and 2017-12-02 00:00:00 INFO:prophet:Applying in parallel with <concurrent.futures.process.ProcessPoolExecutor object at 0x7fdcf81f96d0> INFO:prophet:Making 4 forecasts with cutoffs between 2017-03-07 00:00:00 and 2017-12-02 00:00:00 INFO:prophet:Applying in parallel with <concurrent.futures.process.ProcessPoolExecutor object at 0x7fdcf69ed610> INFO:prophet:Making 4 forecasts with cutoffs between 2017-03-07 00:00:00 and 2017-12-02 00:00:00 INFO:prophet:Applying in parallel with <concurrent.futures.process.ProcessPoolExecutor object at 0x7fdcda26bfd0> INFO:prophet:Making 4 forecasts with cutoffs between 2017-03-07 00:00:00 and 2017-12-02 00:00:00 INFO:prophet:Applying in parallel with <concurrent.futures.process.ProcessPoolExecutor object at 0x7fdcf6a02ad0> INFO:prophet:Making 4 forecasts with cutoffs between 2017-03-07 00:00:00 and 2017-12-02 00:00:00 INFO:prophet:Applying in parallel with <concurrent.futures.process.ProcessPoolExecutor object at 0x7fdcf7af5a10> INFO:prophet:Making 4 forecasts with cutoffs between 2017-03-07 00:00:00 and 2017-12-02 00:00:00 INFO:prophet:Applying in parallel with <concurrent.futures.process.ProcessPoolExecutor object at 0x7fdcf7939150> INFO:prophet:Making 4 forecasts with cutoffs between 2017-03-07 00:00:00 and 2017-12-02 00:00:00 INFO:prophet:Applying in parallel with <concurrent.futures.process.ProcessPoolExecutor object at 0x7fdcd8e463d0> INFO:prophet:Making 4 forecasts with cutoffs between 2017-03-07 00:00:00 and 2017-12-02 00:00:00 INFO:prophet:Applying in parallel with <concurrent.futures.process.ProcessPoolExecutor object at 0x7fdcf82ad850>
changepoint_prior_scale seasonality_prior_scale growth seasonality_mode \
0 0.00 0.01 linear additive
1 0.00 0.01 linear multiplicative
2 0.00 0.01 flat additive
3 0.00 0.01 flat multiplicative
4 0.00 0.10 linear additive
.. ... ... ... ...
59 0.50 1.00 flat multiplicative
60 0.50 10.00 linear additive
61 0.50 10.00 linear multiplicative
62 0.50 10.00 flat additive
63 0.50 10.00 flat multiplicative
daily_seasonality weekly_seasonality mape
0 False False 0.16
1 False False 0.17
2 False False 0.18
3 False False 0.19
4 False False 0.15
.. ... ... ...
59 False False 0.17
60 False False 0.24
61 False False 0.24
62 False False 0.17
63 False False 0.17
[64 rows x 7 columns]
# selecting the best hyperparameters
best_params_conv_Q = all_params[np.argmin(mapes)]
print(best_params_conv_Q)
{'changepoint_prior_scale': 0.001, 'seasonality_prior_scale': 10.0, 'growth': 'linear', 'seasonality_mode': 'multiplicative', 'daily_seasonality': False, 'weekly_seasonality': False}
# creating the model with the best parameters
model_conv_Q = Prophet(**best_params_conv_Q)
# fitting the model on the whole training set
model_conv_Q.fit(conv_Q_df)
WARNING:prophet.models:Optimization terminated abnormally. Falling back to Newton.
<prophet.forecaster.Prophet at 0x7fdcda26b6d0>
future_conv_Q = model_conv_Q.make_future_dataframe(periods=len(prices_US_conv_2019), freq="W")
# forecast visualisation
forecast_conv_Q = model_conv_Q.predict(future_conv_Q)
fig_conv_Q = model_conv_Q.plot(forecast_conv_Q)
# model components visualisation
fig_conv_Q_comp = model_conv_Q.plot_components(forecast_conv_Q)
# cross-validation of the selected model
df_cv_conv_Q = cross_validation(model_conv_Q, initial="730 days", period="90 days", horizon="365 days", parallel="processes")
INFO:prophet:Making 4 forecasts with cutoffs between 2017-03-07 00:00:00 and 2017-12-02 00:00:00 INFO:prophet:Applying in parallel with <concurrent.futures.process.ProcessPoolExecutor object at 0x7fdcdaa397d0> WARNING:prophet.models:Optimization terminated abnormally. Falling back to Newton. WARNING:prophet.models:Optimization terminated abnormally. Falling back to Newton. WARNING:prophet.models:Optimization terminated abnormally. Falling back to Newton.
# MAPE of the cross-validated model visualisation
fig_conv_Q_cv_mape = plot_cross_validation_metric(df_cv_conv_Q, metric='mape')
# selecting only the test period forecast
pred_conv_Q_2019 = forecast_conv_Q[forecast_conv_Q["ds"]>"2019-01-01"][["ds", "yhat"]]
# simplifing the actuals dataframe for comparison
actuals_conv_Q_2019 = prices_US_conv_2019[["Date", "TotalVolume"]]
actuals_conv_Q_2019.columns = ["ds", "y"]
# merging the prediction and actuals dataframes into one
conv_Q_2019 = actuals_conv_Q_2019.merge(pred_conv_Q_2019, on="ds")
# plotting training, test and prediction data
conv_Q_df.set_index("ds")["y"].plot(label="training")
conv_Q_2019.set_index("ds")["y"].plot(label="test")
conv_Q_2019.set_index("ds")["yhat"].plot(label="prediction")
plt.xlabel("Date")
plt.ylabel("Volume")
plt.title("Volume conventional - test vs. prediction")
plt.legend()
plt.savefig(str(constants.OUTPUTS_DIR) + "/Volume conventional - test vs. prediction.png")
plt.show()
# model's MAPE calculation
model_conv_Q_MAPE = MAPE(conv_Q_2019["y"], conv_Q_2019["yhat"])
# default vs. tuned models' MAPE
print("Default model MAPE: {:.2f}".format(model_conv_Q_def_MAPE))
print("Tuned model MAPE: {:.2f}".format(model_conv_Q_MAPE))
Default model MAPE: 13.46 Tuned model MAPE: 13.70
# simplifing the full dataset for use by Prophet
conv_Q_full = prices_US_conv_merged[["Date", "TotalVolume"]]
conv_Q_full.columns = ["ds", "y"]
# re-training the model using the best performing parameters and full dataset
# creating 6-month (= 26-week) forecast
model_conv_Q_for = Prophet(**best_params_conv_Q)
model_conv_Q_for.fit(conv_Q_full)
future_6m_conv_Q = model_conv_Q_for.make_future_dataframe(periods=26, freq="W", include_history=False)
forecast_6m_conv_Q = model_conv_Q_for.predict(future_6m_conv_Q)
fig_6m_conv_Q = model_conv_Q_for.plot(forecast_6m_conv_Q)
WARNING:prophet.models:Optimization terminated abnormally. Falling back to Newton.
# plotting forecast
conv_Q_full.set_index("ds")["y"].plot(label="original")
forecast_6m_conv_Q.set_index("ds")["yhat"].plot(label="forecast")
plt.xlabel("Date")
plt.ylabel("Volume")
plt.title("Volume conventional - forecast")
plt.legend()
plt.savefig(str(constants.OUTPUTS_DIR) + "/Volume conventional - forecast.png")
plt.show()
# simplifing the dataframe for use by Prophet
org_Q_df = prices_US_org_2015_2018[["Date", "TotalVolume"]]
org_Q_df.columns = ["ds", "y"]
# org_Q_df["cap"]
# creating model with default values, skipping only the daily and weekly seasonality which isn't present in our data
model_org_Q_def = Prophet(daily_seasonality=False, weekly_seasonality=False)
# fitting the model on the whole training set
model_org_Q_def.fit(org_Q_df)
<prophet.forecaster.Prophet at 0x7fdcd91063d0>
future_org_Q_def = model_org_Q_def.make_future_dataframe(periods=len(prices_US_org_2019), freq="W")
# forecast visualisation
forecast_org_Q_def = model_org_Q_def.predict(future_org_Q_def)
fig_org_Q_def = model_org_Q_def.plot(forecast_org_Q_def)
# model components visualisation
fig_org_Q_comp_def = model_org_Q_def.plot_components(forecast_org_Q_def)
# selecting only the test period forecast
pred_org_Q_2019_def = forecast_org_Q_def[forecast_org_Q_def["ds"]>"2019-01-01"][["ds", "yhat"]]
# simplifing the actuals dataframe for comparison
actuals_org_Q_2019_def = prices_US_org_2019[["Date", "TotalVolume"]]
actuals_org_Q_2019_def.columns = ["ds", "y"]
# merging the prediction and actuals dataframes into one
org_Q_2019_def = actuals_org_Q_2019_def.merge(pred_org_Q_2019_def, on="ds")
# plotting training, test and prediction data
org_Q_df.set_index("ds")["y"].plot(label="training")
org_Q_2019_def.set_index("ds")["y"].plot(label="test")
org_Q_2019_def.set_index("ds")["yhat"].plot(label="prediction")
plt.xlabel("Date")
plt.ylabel("Volume")
plt.title("Volume organic - default model")
plt.legend()
plt.savefig(str(constants.OUTPUTS_DIR) + "/Volume organic - default model.png")
plt.show()
# model's MAPE calculation
model_org_Q_def_MAPE = MAPE(org_Q_2019_def["y"], org_Q_2019_def["yhat"])
print("Default model MAPE: {:.2f}".format(model_org_Q_def_MAPE))
Default model MAPE: 27.94
# cross-validation hyperparameters on the Prophet model
param_grid = {
"changepoint_prior_scale": [0.001, 0.01, 0.1, 0.5],
"seasonality_prior_scale": [0.01, 0.1, 1.0, 10.0],
"growth": ["linear", "flat"], # "logistic" might be better for this model but we'd need the cap value estimate from the client
"seasonality_mode": ["additive", "multiplicative"],
"daily_seasonality": [False],
"weekly_seasonality": [False]
}
# Generate all combinations of parameters
all_params = [dict(zip(param_grid.keys(), v)) for v in itertools.product(*param_grid.values())]
mapes = [] # Store the MAPEs for each params here
# Use cross validation to evaluate all parameters
for params in all_params:
m = Prophet(**params).fit(org_Q_df) # Fit model with given params
df_cv_org_Q = cross_validation(m, initial="730 days", period="90 days", horizon="365 days", parallel="processes")
df_p_org_Q = performance_metrics(df_cv_org_Q, rolling_window=1)
mapes.append(df_p_org_Q["mape"].values[0])
# Find the best parameters
tuning_results = pd.DataFrame(all_params)
tuning_results["mape"] = mapes
print(tuning_results)
WARNING:prophet.models:Optimization terminated abnormally. Falling back to Newton. INFO:prophet:Making 4 forecasts with cutoffs between 2017-03-07 00:00:00 and 2017-12-02 00:00:00 INFO:prophet:Applying in parallel with <concurrent.futures.process.ProcessPoolExecutor object at 0x7fdcd9cbf210> WARNING:prophet.models:Optimization terminated abnormally. Falling back to Newton. WARNING:prophet.models:Optimization terminated abnormally. Falling back to Newton. WARNING:prophet.models:Optimization terminated abnormally. Falling back to Newton. WARNING:prophet.models:Optimization terminated abnormally. Falling back to Newton. WARNING:prophet.models:Optimization terminated abnormally. Falling back to Newton. INFO:prophet:Making 4 forecasts with cutoffs between 2017-03-07 00:00:00 and 2017-12-02 00:00:00 INFO:prophet:Applying in parallel with <concurrent.futures.process.ProcessPoolExecutor object at 0x7fdcd9cd7c90> WARNING:prophet.models:Optimization terminated abnormally. Falling back to Newton. WARNING:prophet.models:Optimization terminated abnormally. Falling back to Newton. WARNING:prophet.models:Optimization terminated abnormally. Falling back to Newton. WARNING:prophet.models:Optimization terminated abnormally. Falling back to Newton. INFO:prophet:Making 4 forecasts with cutoffs between 2017-03-07 00:00:00 and 2017-12-02 00:00:00 INFO:prophet:Applying in parallel with <concurrent.futures.process.ProcessPoolExecutor object at 0x7fdcd9cd7a10> INFO:prophet:Making 4 forecasts with cutoffs between 2017-03-07 00:00:00 and 2017-12-02 00:00:00 INFO:prophet:Applying in parallel with <concurrent.futures.process.ProcessPoolExecutor object at 0x7fdcdbc95310> WARNING:prophet.models:Optimization terminated abnormally. Falling back to Newton. INFO:prophet:Making 4 forecasts with cutoffs between 2017-03-07 00:00:00 and 2017-12-02 00:00:00 INFO:prophet:Applying in parallel with <concurrent.futures.process.ProcessPoolExecutor object at 0x7fdcdbce4250> WARNING:prophet.models:Optimization terminated abnormally. Falling back to Newton. WARNING:prophet.models:Optimization terminated abnormally. Falling back to Newton. WARNING:prophet.models:Optimization terminated abnormally. Falling back to Newton. WARNING:prophet.models:Optimization terminated abnormally. Falling back to Newton. WARNING:prophet.models:Optimization terminated abnormally. Falling back to Newton. INFO:prophet:Making 4 forecasts with cutoffs between 2017-03-07 00:00:00 and 2017-12-02 00:00:00 INFO:prophet:Applying in parallel with <concurrent.futures.process.ProcessPoolExecutor object at 0x7fdcdbc95490> WARNING:prophet.models:Optimization terminated abnormally. Falling back to Newton. WARNING:prophet.models:Optimization terminated abnormally. Falling back to Newton. WARNING:prophet.models:Optimization terminated abnormally. Falling back to Newton. WARNING:prophet.models:Optimization terminated abnormally. Falling back to Newton. INFO:prophet:Making 4 forecasts with cutoffs between 2017-03-07 00:00:00 and 2017-12-02 00:00:00 INFO:prophet:Applying in parallel with <concurrent.futures.process.ProcessPoolExecutor object at 0x7fdcd9cbf190> INFO:prophet:Making 4 forecasts with cutoffs between 2017-03-07 00:00:00 and 2017-12-02 00:00:00 INFO:prophet:Applying in parallel with <concurrent.futures.process.ProcessPoolExecutor object at 0x7fdcdbc2c490> WARNING:prophet.models:Optimization terminated abnormally. Falling back to Newton. INFO:prophet:Making 4 forecasts with cutoffs between 2017-03-07 00:00:00 and 2017-12-02 00:00:00 INFO:prophet:Applying in parallel with <concurrent.futures.process.ProcessPoolExecutor object at 0x7fdcdbbe8090> WARNING:prophet.models:Optimization terminated abnormally. Falling back to Newton. WARNING:prophet.models:Optimization terminated abnormally. Falling back to Newton. WARNING:prophet.models:Optimization terminated abnormally. Falling back to Newton. WARNING:prophet.models:Optimization terminated abnormally. Falling back to Newton. WARNING:prophet.models:Optimization terminated abnormally. Falling back to Newton. INFO:prophet:Making 4 forecasts with cutoffs between 2017-03-07 00:00:00 and 2017-12-02 00:00:00 INFO:prophet:Applying in parallel with <concurrent.futures.process.ProcessPoolExecutor object at 0x7fdcdbc36d10> WARNING:prophet.models:Optimization terminated abnormally. Falling back to Newton. WARNING:prophet.models:Optimization terminated abnormally. Falling back to Newton. WARNING:prophet.models:Optimization terminated abnormally. Falling back to Newton. WARNING:prophet.models:Optimization terminated abnormally. Falling back to Newton. INFO:prophet:Making 4 forecasts with cutoffs between 2017-03-07 00:00:00 and 2017-12-02 00:00:00 INFO:prophet:Applying in parallel with <concurrent.futures.process.ProcessPoolExecutor object at 0x7fdcdbc6d710> INFO:prophet:Making 4 forecasts with cutoffs between 2017-03-07 00:00:00 and 2017-12-02 00:00:00 INFO:prophet:Applying in parallel with <concurrent.futures.process.ProcessPoolExecutor object at 0x7fdcdbc81d10> WARNING:prophet.models:Optimization terminated abnormally. Falling back to Newton. INFO:prophet:Making 4 forecasts with cutoffs between 2017-03-07 00:00:00 and 2017-12-02 00:00:00 INFO:prophet:Applying in parallel with <concurrent.futures.process.ProcessPoolExecutor object at 0x7fdcdbbd4fd0> WARNING:prophet.models:Optimization terminated abnormally. Falling back to Newton. WARNING:prophet.models:Optimization terminated abnormally. Falling back to Newton. WARNING:prophet.models:Optimization terminated abnormally. Falling back to Newton. WARNING:prophet.models:Optimization terminated abnormally. Falling back to Newton. WARNING:prophet.models:Optimization terminated abnormally. Falling back to Newton. INFO:prophet:Making 4 forecasts with cutoffs between 2017-03-07 00:00:00 and 2017-12-02 00:00:00 INFO:prophet:Applying in parallel with <concurrent.futures.process.ProcessPoolExecutor object at 0x7fdcdbc6d710> WARNING:prophet.models:Optimization terminated abnormally. Falling back to Newton. WARNING:prophet.models:Optimization terminated abnormally. Falling back to Newton. WARNING:prophet.models:Optimization terminated abnormally. Falling back to Newton. WARNING:prophet.models:Optimization terminated abnormally. Falling back to Newton. INFO:prophet:Making 4 forecasts with cutoffs between 2017-03-07 00:00:00 and 2017-12-02 00:00:00 INFO:prophet:Applying in parallel with <concurrent.futures.process.ProcessPoolExecutor object at 0x7fdcdb6d2250> INFO:prophet:Making 4 forecasts with cutoffs between 2017-03-07 00:00:00 and 2017-12-02 00:00:00 INFO:prophet:Applying in parallel with <concurrent.futures.process.ProcessPoolExecutor object at 0x7fdcd9ce3550> INFO:prophet:Making 4 forecasts with cutoffs between 2017-03-07 00:00:00 and 2017-12-02 00:00:00 INFO:prophet:Applying in parallel with <concurrent.futures.process.ProcessPoolExecutor object at 0x7fdcdb304c10> INFO:prophet:Making 4 forecasts with cutoffs between 2017-03-07 00:00:00 and 2017-12-02 00:00:00 INFO:prophet:Applying in parallel with <concurrent.futures.process.ProcessPoolExecutor object at 0x7fdcdbbde950> INFO:prophet:Making 4 forecasts with cutoffs between 2017-03-07 00:00:00 and 2017-12-02 00:00:00 INFO:prophet:Applying in parallel with <concurrent.futures.process.ProcessPoolExecutor object at 0x7fdcdbce4950> INFO:prophet:Making 4 forecasts with cutoffs between 2017-03-07 00:00:00 and 2017-12-02 00:00:00 INFO:prophet:Applying in parallel with <concurrent.futures.process.ProcessPoolExecutor object at 0x7fdcdb6e0a90> INFO:prophet:Making 4 forecasts with cutoffs between 2017-03-07 00:00:00 and 2017-12-02 00:00:00 INFO:prophet:Applying in parallel with <concurrent.futures.process.ProcessPoolExecutor object at 0x7fdcdbbe8610> INFO:prophet:Making 4 forecasts with cutoffs between 2017-03-07 00:00:00 and 2017-12-02 00:00:00 INFO:prophet:Applying in parallel with <concurrent.futures.process.ProcessPoolExecutor object at 0x7fdcdb304910> INFO:prophet:Making 4 forecasts with cutoffs between 2017-03-07 00:00:00 and 2017-12-02 00:00:00 INFO:prophet:Applying in parallel with <concurrent.futures.process.ProcessPoolExecutor object at 0x7fdcdbbde9d0> INFO:prophet:Making 4 forecasts with cutoffs between 2017-03-07 00:00:00 and 2017-12-02 00:00:00 INFO:prophet:Applying in parallel with <concurrent.futures.process.ProcessPoolExecutor object at 0x7fdcdb30fe50> INFO:prophet:Making 4 forecasts with cutoffs between 2017-03-07 00:00:00 and 2017-12-02 00:00:00 INFO:prophet:Applying in parallel with <concurrent.futures.process.ProcessPoolExecutor object at 0x7fdcdbc1f6d0> INFO:prophet:Making 4 forecasts with cutoffs between 2017-03-07 00:00:00 and 2017-12-02 00:00:00 INFO:prophet:Applying in parallel with <concurrent.futures.process.ProcessPoolExecutor object at 0x7fdcdbc09ad0> INFO:prophet:Making 4 forecasts with cutoffs between 2017-03-07 00:00:00 and 2017-12-02 00:00:00 INFO:prophet:Applying in parallel with <concurrent.futures.process.ProcessPoolExecutor object at 0x7fdcdb30f950> INFO:prophet:Making 4 forecasts with cutoffs between 2017-03-07 00:00:00 and 2017-12-02 00:00:00 INFO:prophet:Applying in parallel with <concurrent.futures.process.ProcessPoolExecutor object at 0x7fdcdb283fd0> INFO:prophet:Making 4 forecasts with cutoffs between 2017-03-07 00:00:00 and 2017-12-02 00:00:00 INFO:prophet:Applying in parallel with <concurrent.futures.process.ProcessPoolExecutor object at 0x7fdcdb6e0910> INFO:prophet:Making 4 forecasts with cutoffs between 2017-03-07 00:00:00 and 2017-12-02 00:00:00 INFO:prophet:Applying in parallel with <concurrent.futures.process.ProcessPoolExecutor object at 0x7fdcdb6bd850> INFO:prophet:Making 4 forecasts with cutoffs between 2017-03-07 00:00:00 and 2017-12-02 00:00:00 INFO:prophet:Applying in parallel with <concurrent.futures.process.ProcessPoolExecutor object at 0x7fdcdbbf4650> INFO:prophet:Making 4 forecasts with cutoffs between 2017-03-07 00:00:00 and 2017-12-02 00:00:00 INFO:prophet:Applying in parallel with <concurrent.futures.process.ProcessPoolExecutor object at 0x7fdcdbc81a90> INFO:prophet:Making 4 forecasts with cutoffs between 2017-03-07 00:00:00 and 2017-12-02 00:00:00 INFO:prophet:Applying in parallel with <concurrent.futures.process.ProcessPoolExecutor object at 0x7fdcd90c0590> INFO:prophet:Making 4 forecasts with cutoffs between 2017-03-07 00:00:00 and 2017-12-02 00:00:00 INFO:prophet:Applying in parallel with <concurrent.futures.process.ProcessPoolExecutor object at 0x7fdcf7adeed0> INFO:prophet:Making 4 forecasts with cutoffs between 2017-03-07 00:00:00 and 2017-12-02 00:00:00 INFO:prophet:Applying in parallel with <concurrent.futures.process.ProcessPoolExecutor object at 0x7fdcdb6e93d0> INFO:prophet:Making 4 forecasts with cutoffs between 2017-03-07 00:00:00 and 2017-12-02 00:00:00 INFO:prophet:Applying in parallel with <concurrent.futures.process.ProcessPoolExecutor object at 0x7fdcdbbde9d0> INFO:prophet:Making 4 forecasts with cutoffs between 2017-03-07 00:00:00 and 2017-12-02 00:00:00 INFO:prophet:Applying in parallel with <concurrent.futures.process.ProcessPoolExecutor object at 0x7fdcdb2cf9d0> INFO:prophet:Making 4 forecasts with cutoffs between 2017-03-07 00:00:00 and 2017-12-02 00:00:00 INFO:prophet:Applying in parallel with <concurrent.futures.process.ProcessPoolExecutor object at 0x7fdcdbd19d50> INFO:prophet:Making 4 forecasts with cutoffs between 2017-03-07 00:00:00 and 2017-12-02 00:00:00 INFO:prophet:Applying in parallel with <concurrent.futures.process.ProcessPoolExecutor object at 0x7fdcdb2ada10> INFO:prophet:Making 4 forecasts with cutoffs between 2017-03-07 00:00:00 and 2017-12-02 00:00:00 INFO:prophet:Applying in parallel with <concurrent.futures.process.ProcessPoolExecutor object at 0x7fdcdb2a33d0> INFO:prophet:Making 4 forecasts with cutoffs between 2017-03-07 00:00:00 and 2017-12-02 00:00:00 INFO:prophet:Applying in parallel with <concurrent.futures.process.ProcessPoolExecutor object at 0x7fdcf7aa2fd0> INFO:prophet:Making 4 forecasts with cutoffs between 2017-03-07 00:00:00 and 2017-12-02 00:00:00 INFO:prophet:Applying in parallel with <concurrent.futures.process.ProcessPoolExecutor object at 0x7fdcf59a8950> INFO:prophet:Making 4 forecasts with cutoffs between 2017-03-07 00:00:00 and 2017-12-02 00:00:00 INFO:prophet:Applying in parallel with <concurrent.futures.process.ProcessPoolExecutor object at 0x7fdcd9d12490> INFO:prophet:Making 4 forecasts with cutoffs between 2017-03-07 00:00:00 and 2017-12-02 00:00:00 INFO:prophet:Applying in parallel with <concurrent.futures.process.ProcessPoolExecutor object at 0x7fdcd9d24290> INFO:prophet:Making 4 forecasts with cutoffs between 2017-03-07 00:00:00 and 2017-12-02 00:00:00 INFO:prophet:Applying in parallel with <concurrent.futures.process.ProcessPoolExecutor object at 0x7fdcdb6c7c10> INFO:prophet:Making 4 forecasts with cutoffs between 2017-03-07 00:00:00 and 2017-12-02 00:00:00 INFO:prophet:Applying in parallel with <concurrent.futures.process.ProcessPoolExecutor object at 0x7fdcdb6bd250> INFO:prophet:Making 4 forecasts with cutoffs between 2017-03-07 00:00:00 and 2017-12-02 00:00:00 INFO:prophet:Applying in parallel with <concurrent.futures.process.ProcessPoolExecutor object at 0x7fdcdb2da0d0> INFO:prophet:Making 4 forecasts with cutoffs between 2017-03-07 00:00:00 and 2017-12-02 00:00:00 INFO:prophet:Applying in parallel with <concurrent.futures.process.ProcessPoolExecutor object at 0x7fdcdb2ad890> INFO:prophet:Making 4 forecasts with cutoffs between 2017-03-07 00:00:00 and 2017-12-02 00:00:00 INFO:prophet:Applying in parallel with <concurrent.futures.process.ProcessPoolExecutor object at 0x7fdcf615b590> INFO:prophet:Making 4 forecasts with cutoffs between 2017-03-07 00:00:00 and 2017-12-02 00:00:00 INFO:prophet:Applying in parallel with <concurrent.futures.process.ProcessPoolExecutor object at 0x7fdcd9d37150> INFO:prophet:Making 4 forecasts with cutoffs between 2017-03-07 00:00:00 and 2017-12-02 00:00:00 INFO:prophet:Applying in parallel with <concurrent.futures.process.ProcessPoolExecutor object at 0x7fdcd936a110> INFO:prophet:Making 4 forecasts with cutoffs between 2017-03-07 00:00:00 and 2017-12-02 00:00:00 INFO:prophet:Applying in parallel with <concurrent.futures.process.ProcessPoolExecutor object at 0x7fdcdb2da490> INFO:prophet:Making 4 forecasts with cutoffs between 2017-03-07 00:00:00 and 2017-12-02 00:00:00 INFO:prophet:Applying in parallel with <concurrent.futures.process.ProcessPoolExecutor object at 0x7fdcd9377190> INFO:prophet:Making 4 forecasts with cutoffs between 2017-03-07 00:00:00 and 2017-12-02 00:00:00 INFO:prophet:Applying in parallel with <concurrent.futures.process.ProcessPoolExecutor object at 0x7fdcd9377e90> INFO:prophet:Making 4 forecasts with cutoffs between 2017-03-07 00:00:00 and 2017-12-02 00:00:00 INFO:prophet:Applying in parallel with <concurrent.futures.process.ProcessPoolExecutor object at 0x7fdcd9d37150> INFO:prophet:Making 4 forecasts with cutoffs between 2017-03-07 00:00:00 and 2017-12-02 00:00:00 INFO:prophet:Applying in parallel with <concurrent.futures.process.ProcessPoolExecutor object at 0x7fdcf5355650> INFO:prophet:Making 4 forecasts with cutoffs between 2017-03-07 00:00:00 and 2017-12-02 00:00:00 INFO:prophet:Applying in parallel with <concurrent.futures.process.ProcessPoolExecutor object at 0x7fdcd9d24d90> INFO:prophet:Making 4 forecasts with cutoffs between 2017-03-07 00:00:00 and 2017-12-02 00:00:00 INFO:prophet:Applying in parallel with <concurrent.futures.process.ProcessPoolExecutor object at 0x7fdcd9d17290> INFO:prophet:Making 4 forecasts with cutoffs between 2017-03-07 00:00:00 and 2017-12-02 00:00:00 INFO:prophet:Applying in parallel with <concurrent.futures.process.ProcessPoolExecutor object at 0x7fdcdb31aa90> INFO:prophet:Making 4 forecasts with cutoffs between 2017-03-07 00:00:00 and 2017-12-02 00:00:00 INFO:prophet:Applying in parallel with <concurrent.futures.process.ProcessPoolExecutor object at 0x7fdcdb315110> INFO:prophet:Making 4 forecasts with cutoffs between 2017-03-07 00:00:00 and 2017-12-02 00:00:00 INFO:prophet:Applying in parallel with <concurrent.futures.process.ProcessPoolExecutor object at 0x7fdcdbc37890> INFO:prophet:Making 4 forecasts with cutoffs between 2017-03-07 00:00:00 and 2017-12-02 00:00:00 INFO:prophet:Applying in parallel with <concurrent.futures.process.ProcessPoolExecutor object at 0x7fdcdb260650> INFO:prophet:Making 4 forecasts with cutoffs between 2017-03-07 00:00:00 and 2017-12-02 00:00:00 INFO:prophet:Applying in parallel with <concurrent.futures.process.ProcessPoolExecutor object at 0x7fdcdb277590> INFO:prophet:Making 4 forecasts with cutoffs between 2017-03-07 00:00:00 and 2017-12-02 00:00:00 INFO:prophet:Applying in parallel with <concurrent.futures.process.ProcessPoolExecutor object at 0x7fdcf530fb90>
changepoint_prior_scale seasonality_prior_scale growth seasonality_mode \
0 0.00 0.01 linear additive
1 0.00 0.01 linear multiplicative
2 0.00 0.01 flat additive
3 0.00 0.01 flat multiplicative
4 0.00 0.10 linear additive
.. ... ... ... ...
59 0.50 1.00 flat multiplicative
60 0.50 10.00 linear additive
61 0.50 10.00 linear multiplicative
62 0.50 10.00 flat additive
63 0.50 10.00 flat multiplicative
daily_seasonality weekly_seasonality mape
0 False False 0.12
1 False False 0.14
2 False False 0.36
3 False False 0.35
4 False False 0.11
.. ... ... ...
59 False False 0.38
60 False False 0.27
61 False False 0.22
62 False False 0.38
63 False False 0.38
[64 rows x 7 columns]
# selecting the best hyperparameters
best_params_org_Q = all_params[np.argmin(mapes)]
print(best_params_org_Q)
{'changepoint_prior_scale': 0.001, 'seasonality_prior_scale': 10.0, 'growth': 'linear', 'seasonality_mode': 'additive', 'daily_seasonality': False, 'weekly_seasonality': False}
# creating the model with the best parameters
model_org_Q = Prophet(**best_params_org_Q)
# fitting the model on the whole training set
model_org_Q.fit(org_Q_df)
WARNING:prophet.models:Optimization terminated abnormally. Falling back to Newton.
<prophet.forecaster.Prophet at 0x7fdcdbcb8b50>
future_org_Q = model_org_Q.make_future_dataframe(periods=len(prices_US_org_2019), freq="W")
# forecast visualisation
forecast_org_Q = model_org_Q.predict(future_org_Q)
fig_org_Q = model_org_Q.plot(forecast_org_Q)
# model components visualisation
fig_org_Q_comp = model_org_Q.plot_components(forecast_org_Q)
# cross-validation of the selected model
df_cv_org_Q = cross_validation(model_org_Q, initial="730 days", period="90 days", horizon="365 days", parallel="processes")
INFO:prophet:Making 4 forecasts with cutoffs between 2017-03-07 00:00:00 and 2017-12-02 00:00:00 INFO:prophet:Applying in parallel with <concurrent.futures.process.ProcessPoolExecutor object at 0x7fdcd8ec8b50> WARNING:prophet.models:Optimization terminated abnormally. Falling back to Newton. WARNING:prophet.models:Optimization terminated abnormally. Falling back to Newton. WARNING:prophet.models:Optimization terminated abnormally. Falling back to Newton. WARNING:prophet.models:Optimization terminated abnormally. Falling back to Newton.
# MAPE of the cross-validated model visualisation
fig_org_Q_cv_mape = plot_cross_validation_metric(df_cv_org_Q, metric='mape')
# selecting only the test period forecast
pred_org_Q_2019 = forecast_org_Q[forecast_org_Q["ds"]>"2019-01-01"][["ds", "yhat"]]
# simplifing the actuals dataframe for comparison
actuals_org_Q_2019 = prices_US_org_2019[["Date", "TotalVolume"]]
actuals_org_Q_2019.columns = ["ds", "y"]
# merging the prediction and actuals dataframes into one
org_Q_2019 = actuals_org_Q_2019.merge(pred_org_Q_2019, on="ds")
# plotting training, test and prediction data
org_Q_df.set_index("ds")["y"].plot(label="training")
org_Q_2019.set_index("ds")["y"].plot(label="test")
org_Q_2019.set_index("ds")["yhat"].plot(label="prediction")
plt.xlabel("Date")
plt.ylabel("Volume")
plt.title("Volume organic - test vs. prediction")
plt.legend()
plt.savefig(str(constants.OUTPUTS_DIR) + "/Volume organic - test vs. prediction.png")
plt.show()
# model's MAPE calculation
model_org_Q_MAPE = MAPE(org_Q_2019["y"], org_Q_2019["yhat"])
# default vs. tuned models' MAPE
print("Default model MAPE: {:.2f}".format(model_org_Q_def_MAPE))
print("Tuned model MAPE: {:.2f}".format(model_org_Q_MAPE))
Default model MAPE: 27.94 Tuned model MAPE: 23.18
# simplifing the full dataset for use by Prophet
org_Q_full = prices_US_org_merged[["Date", "TotalVolume"]]
org_Q_full.columns = ["ds", "y"]
# re-training the model using the best performing parameters and full dataset
# creating 6-month (= 26-week) forecast
model_org_Q_for = Prophet(**best_params_org_Q)
model_org_Q_for.fit(org_Q_full)
future_6m_org_Q = model_org_Q_for.make_future_dataframe(periods=26, freq="W", include_history=False)
forecast_6m_org_Q = model_org_Q_for.predict(future_6m_org_Q)
fig_6m_org_Q = model_org_Q_for.plot(forecast_6m_org_Q)
WARNING:prophet.models:Optimization terminated abnormally. Falling back to Newton.
# plotting forecast
org_Q_full.set_index("ds")["y"].plot(label="original")
forecast_6m_org_Q.set_index("ds")["yhat"].plot(label="forecast")
plt.xlabel("Date")
plt.ylabel("Volume")
plt.title("Volume organic - forecast")
plt.legend()
plt.savefig(str(constants.OUTPUTS_DIR) + "/Volume organic - forecast.png")
plt.show()